Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kaggle CI: dispatch the kernel and collect it later, instead of holding a runner (#10183)
Daniel Han6bc095ae0846
diff --git a/.github/scripts/kaggle_studio_ci/collect_evidence.py b/.github/scripts/kaggle_studio_ci/collect_evidence.pyindex 20525e85fdc..a798983f5c9 100644--- a/.github/scripts/kaggle_studio_ci/collect_evidence.py+++ b/.github/scripts/kaggle_studio_ci/collect_evidence.py@@ -58,11 +58,19 @@ def iter_text(path: Path):nb = json.loads(nb_path.read_text(encoding = "utf-8", errors = "replace"))except Exception: # noqa: BLE001continue- for cell in nb.get("cells", []):- for output in cell.get("outputs", []):+ # Kernels have returned valid JSON that is not a notebook (`[]`, or a+ # non-object cell); raising here fails the job after the verdict posted.+ if not isinstance(nb, dict):+ continue+ for cell in nb.get("cells") or []:+ if not isinstance(cell, dict):+ continue+ for output in cell.get("outputs") or []:+ if not isinstance(output, dict):+ continuetext = output.get("text") or ""if isinstance(text, list):- text = "".join(text)+ text = "".join(str(t) for t in text)yield textfor log_path in sorted(path.rglob("kernel.log")):raw = log_path.read_text(encoding = "utf-8", errors = "replace")diff --git a/.github/scripts/kaggle_t4_ci/BUDGET.md b/.github/scripts/kaggle_t4_ci/BUDGET.mdindex 0717c1bef07..f7c6c174521 100644--- a/.github/scripts/kaggle_t4_ci/BUDGET.md+++ b/.github/scripts/kaggle_t4_ci/BUDGET.md@@ -146,6 +146,14 @@ reserves against its own budget rather than this one.from `launch.py` and `--kernels`; do not edit the number here or in theworkflow without changing what it is derived from.+The Studio leg's `--budget-hours` covers the reaper window, not its 70-minute+kernel ceiling: nothing waits on the kernel now, and Kaggle has been caught+leaving one RUNNING two hours past its own timeout. The collector reaps at+`DEFAULT_MAX_AGE_HOURS` (3h) plus its 10-minute schedule and 30-minute job+timeout, so 3.67h is the worst case one dispatch can bill and the budget is 4,+not 2. `test_every_gate_budget_covers_the_reaper_window` recomputes it from+the collector workflow for both legs.+## What would change these numbers* A second Kaggle account doubles supply and the percentage could roughlydiff --git a/.github/scripts/kaggle_t4_ci/build_kernel.py b/.github/scripts/kaggle_t4_ci/build_kernel.pyindex c17ee5daff3..9d55f0e8163 100644--- a/.github/scripts/kaggle_t4_ci/build_kernel.py+++ b/.github/scripts/kaggle_t4_ci/build_kernel.py@@ -589,9 +589,14 @@ def build_driver(shared_wheel_specs: tuple[str, ...] = (),overlays: dict[str, tuple[str, ...]] | None = None,all_card: tuple[str, ...] = (),+ expected_reports: int | None = None,) -> dict:"""Kernel notebook that runs the payloads across the session's GPUs.+ ``expected_reports`` is how many T4_SMOKE_REPORT lines to expect, echoed+ into the kernel log so collect.py can judge the kernel days later. Defaults+ to one per payload, less the report-less Studio install half.+``isolation`` maps a payload to whether its virtualenv may see the Kaggleimage's site-packages. Per payload, not per kernel, because legs sharing akernel do not share an answer: see ``Leg.system_site_packages``.@@ -649,6 +654,8 @@ def build_driver(_prefetch_builder().prefetch_cell(list(prefetch_repos)).encode("utf-8"))+ if expected_reports is None:+ expected_reports = len(payloads) - (1 if cpu_lane else 0)setup = f"""import base64, gzip, json, os, pathlib, subprocess, sys, threading, timeprint("{DRIVER_SENTINEL} start", flush=True)@@ -811,6 +818,7 @@ if N_GPU < EXPECTED_GPUS:for name, blob in PAYLOADS.items():(WORK / name).write_bytes(gzip.decompress(base64.b64decode(blob)))print("{DRIVER_SENTINEL}_PAYLOADS " + json.dumps(sorted(PAYLOADS)), flush=True)+print("{DRIVER_SENTINEL}_EXPECT " + json.dumps({{"reports": {expected_reports}}}), flush=True)"""runner = f'''results = {{}}diff --git a/.github/scripts/kaggle_t4_ci/collect.py b/.github/scripts/kaggle_t4_ci/collect.pynew file mode 100644index 00000000000..8edbe9c2a1e--- /dev/null+++ b/.github/scripts/kaggle_t4_ci/collect.py@@ -0,0 +1,746 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Collect the kernels an earlier job dispatched and left running.++``launch.py --dispatch`` pushes a kernel and exits within minutes instead of+holding a runner for the forty Kaggle takes. That trade is only sound if+something else finishes the job, and this is that something.++THE ACCOUNT IS THE QUEUE. No database, artifact or branch sits between the+dispatching job and this one, because each is a second place for the truth to+go stale. ``kernels list --mine`` enumerates the kernels and the SLUG carries+the commit and workflow (``launch.slug_name``), so this script rediscovers+everything from the account and can run from any job, in any order, any number+of times.++Three outcomes per kernel, and they are not symmetric:++* **terminal** -- download the evidence, judge the reports, DELETE, and emit a+ commit status for the sha in the slug. The only path that produces a verdict.+* **still running, within its age ceiling** -- left completely alone; a kernel+ doing its job is not a problem to report.+* **still running, past the ceiling** -- deleted, and reported as a failure to+ collect rather than of the code. This is the reaper, and why the script runs+ on a SCHEDULE: a dispatched kernel bills accelerator quota to its ceiling and+ a quiet repo would leave it there.++This script never talks to GitHub. It emits the statuses it wants posted into+``collect_result.json`` and the workflow posts them (``post_statuses.py``): a+process holding both a Kaggle and a GitHub credential is one bug away from+sending one to the other.++DELETION COMES LAST, after the status is delivered: ``--no-delete``, post, then+``--delete-collected``. Deleting first would turn a transient API failure into+the permanent loss of a verdict.++SAFETY above all. This enumerates a whole ACCOUNT shared with a human and+DELETES what it collects, so nothing is touched unless ``launch.parse_slug``+recognises its name. An unrecognised kernel is invisible, not warned about.+"""++from __future__ import annotations++import argparse+import json+import os+import socket+import sys+import time+from datetime import datetime, timezone+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))++import launch # noqa: E402+from gate import GONE_MARKERS, _as_naive_utc # noqa: E402++# How long a dispatched kernel may stay in flight before the reaper takes it.+# The ceiling is ours because Kaggle's has been caught failing: a kernel pushed+# with `-t 5400` whose nbconvert crashed sat RUNNING two hours past it.+#+# Generous rather than tight: the wired notebook kernel measured 2101.8s, the+# two-account run 41.5 min of job wall clock, and queueing behind another of+# ours adds most of that again. Reaping a kernel that would have finished costs+# a whole run and reports a failure nobody can act on.+DEFAULT_MAX_AGE_HOURS = 3.0++# Paging for the account walk. Not bounded by the reap ceiling (a kernel far+# past it is what the reaper exists for), nor by a small page count: the+# account is SHARED and human kernels are never deleted, so once enough newer+# records sit above an uncollected kernel of ours a fixed cap would never reach+# it again. The walk stops at a page entirely older than LISTING_HORIZON_HOURS+# with none of ours on it, at the pass deadline, or at MAX_PAGES.+PAGE_SIZE = 100+MAX_PAGES = 50+# Kaggle kills a session at 12h whatever else fails, so a kernel of ours that+# last ran before this horizon is neither running nor owed a timely status.+LISTING_HORIZON_HOURS = 24.0 * 7++# The driver's own record of how many payload reports the kernel was BUILT to+# produce, and the only one that survives dispatch: judging a five-payload+# kernel against `--expect 1` turns a run that lost four of them into a pass.+EXPECT_SENTINEL = "KAGGLE_T4_CI_DRIVER_EXPECT "++# Wall clock for the whole collection (evidence downloads dominate). It runs on+# a schedule, so a kernel missed this pass is reached five minutes later.+BUDGET_SEC = 900++# Per-kernel evidence budget, well under the total, so one slow download cannot+# consume the pass and starve every kernel behind it.+EVIDENCE_BUDGET_SEC = 300++SOCKET_TIMEOUT_SEC = 120++# The commit status context per workflow kind. A PUBLIC interface: branch+# protection is configured against these strings, so renaming one silently+# stops requiring the check it names.+STATUS_CONTEXTS = {+ "notebook": "kaggle-t4-notebook",+ "studio": "kaggle-studio-gpu",+}+++def _log(msg: str) -> None:+ print(f"[collect] {msg}", flush = True)+++def _out(key: str, value: str) -> None:+ path = os.environ.get("GITHUB_OUTPUT")+ if not path:+ return+ try:+ with open(path, "a", encoding = "utf-8") as fh:+ fh.write(f"{key}={value}\n")+ except OSError:+ pass+++def kernel_age_hours(kernel, now: datetime) -> float | None:+ """How long ago this kernel last started, or None if Kaggle did not say.++ For an UNFINISHED kernel ``last_run_time`` is when the run started, which is+ what the reap ceiling measures against. None rather than a default, treated+ as too young to reap: guessing old deletes a live session.+ """+ last_run = _as_naive_utc(getattr(kernel, "last_run_time", None))+ if last_run is None:+ return None+ return (now - last_run).total_seconds() / 3600.0+++def find_ours(+ api,+ now: datetime | None = None,+ max_age_hours: float = DEFAULT_MAX_AGE_HOURS,+ page_size: int = PAGE_SIZE,+ max_pages: int = MAX_PAGES,+ horizon_hours: float = LISTING_HORIZON_HOURS,+ deadline: float | None = None,+) -> list[dict]:+ """Every kernel on this account that WE pushed, newest first.++ The filter is ``launch.parse_slug`` and nothing else: an unrecognised kernel+ is not ours to read, let alone delete. ``max_age_hours`` is the reap ceiling+ the caller applies later, not a listing cutoff, because the kernel that most+ needs collecting is the one a delayed pass finds hours past it. The walk+ ends at the first page entirely older than ``horizon_hours`` carrying none+ of ours, at ``deadline``, or at ``max_pages``.+ """+ now = now or datetime.now(timezone.utc).replace(tzinfo = None)+ found: list[dict] = []+ for page in range(1, max_pages + 1):+ if deadline is not None and time.time() >= deadline:+ _log(f"listing stopped at page {page}: out of budget")+ break+ kernels = (+ api.kernels_list(mine = True, page = page, page_size = page_size, sort_by = "dateRun") or []+ )+ if not kernels:+ break+ ours_on_page = 0+ beyond_horizon = 0+ for kernel in kernels:+ age = kernel_age_hours(kernel, now)+ if age is not None and age > horizon_hours:+ beyond_horizon += 1+ ref = getattr(kernel, "ref", None)+ if not ref:+ continue+ parsed = launch.parse_slug(ref)+ if parsed is None:+ continue+ ours_on_page += 1+ found.append(+ {+ "slug": ref,+ "sha": parsed["sha"],+ "kind": parsed["kind"],+ "slot": parsed.get("slot", "1"),+ "legacy": parsed["legacy"],+ "age_hours": age,+ }+ )+ if ours_on_page == 0 and beyond_horizon == len(kernels):+ break+ return found+++def _status_of(api, slug: str) -> str:+ """The kernel's state, with a deleted kernel distinguished from an unknown one.++ A kernel deleted moments ago still lists but 404s its status call:+ definitively not running, not unreadable. Conflating the two makes the+ reaper attack gone kernels and read a genuine outage as idle.+ """+ try:+ raw = str(getattr(api.kernels_status(slug), "status", ""))+ except Exception as exc: # noqa: BLE001+ text = f"{exc}".lower()+ if any(m in text for m in GONE_MARKERS):+ return "GONE"+ _log(f"status unreadable for {slug}: {type(exc).__name__}")+ return "UNKNOWN"+ match = launch._STATUS_RE.search(raw)+ return match.group("status") if match else (raw.strip().upper() or "UNKNOWN")+++def verdict_of(reports: list[dict], expect: int) -> tuple[str, str]:+ """Turn the payload reports into a verdict, exactly as launch.py does inline.++ Kept identical so a dispatched run and a waited-on run reach the same+ conclusion from the same evidence.+ """+ if not reports:+ return "infra", (+ "the kernel finished but produced no payload report; "+ "nothing was learned about the code under test"+ )+ failing = [r for r in reports if not r.get("passed")]+ if failing:+ names = ", ".join(str(r.get("payload") or "?") for r in failing)+ return (+ "fail",+ f"{len(failing)} of {len(reports)} payload(s) failed their assertions: {names}",+ )+ if len(reports) < expect:+ return "partial", (+ f"only {len(reports)} of {expect} payload(s) reported back, "+ "so this is half a comparison rather than a result"+ )+ return "pass", f"all {len(reports)} payload(s) passed"+++def _gone(exc: BaseException) -> bool:+ """Did this failure say the kernel is no longer there?++ Three unserialised collectors run, so another can delete a kernel between+ this pass seeing it terminal and downloading it. Not an infra failure: a+ result somebody else already posted.+ """+ text = f"{exc}".lower()+ return any(m in text for m in GONE_MARKERS)+++def _evidence_lines(dest: Path):+ """Every line of driver output in the collected evidence, notebooks first."""+ for nb_path in sorted(dest.rglob(f"*{launch.OUTPUT_SUFFIX}")):+ try:+ nb = json.loads(nb_path.read_text(encoding = "utf-8", errors = "replace"))+ except Exception: # noqa: BLE001+ continue+ # Valid JSON that is not a notebook is skipped, not raised on: this+ # runs outside the report guard, and an exception here writes no result+ # file and wedges every later pass on this kernel.+ if not isinstance(nb, dict):+ continue+ for cell in nb.get("cells") or []:+ if not isinstance(cell, dict):+ continue+ for output in cell.get("outputs") or []:+ if not isinstance(output, dict):+ continue+ text = output.get("text") or ""+ if isinstance(text, list):+ text = "".join(str(t) for t in text)+ if isinstance(text, str):+ yield from text.splitlines()+ for log_path in sorted(dest.rglob("kernel.log")):+ raw = log_path.read_text(encoding = "utf-8", errors = "replace")+ yield from launch.flatten_kernel_log(raw).splitlines()+++def expected_reports(+ dest: Path,+ default: int,+ kind: str = "",+) -> int | None:+ """How many payload reports this kernel was built to produce.++ Read off the kernel's own evidence, the only record that survives dispatch.+ A kernel predating the sentinel, or a Studio kernel (always one report),+ answers ``default``.+ """+ for line in _evidence_lines(dest):+ if not line.startswith(EXPECT_SENTINEL):+ continue+ try:+ parsed = json.loads(line[len(EXPECT_SENTINEL) :].strip())+ except ValueError:+ continue+ reports = parsed.get("reports") if isinstance(parsed, dict) else None+ if isinstance(reports, int) and reports > 0:+ return reports+ # Only a notebook kernel's own record says how many payloads it carries.+ # Without it the plan is UNKNOWN, not the caller's default, or a kernel that+ # lost four legs reads as a pass. Studio kernels always carry exactly one.+ if kind == "notebook":+ return None+ return default+++# How a verdict is reported to GitHub. `infra` and `partial` are deliberately+# NOT failures: nothing was learned about the code, and a red the author cannot+# act on is how a required check gets dropped from branch protection.+VERDICT_STATE = {+ "pass": "success",+ "fail": "failure",+ "partial": "success",+ "infra": "success",+ "reaped": "failure",+}+++def collect_one(+ api,+ entry: dict,+ outdir: Path,+ expect: int,+ max_age_hours: float,+ delete: bool = True,+ deadline: float | None = None,+) -> dict:+ """Read one kernel to a conclusion. Returns the record for the result file.++ ``deadline`` is the whole pass's; the evidence download gets its own budget+ clamped to what is left, so the last kernel cannot overrun the job.+ """+ slug = entry["slug"]+ state = _status_of(api, slug)+ record = {+ **entry,+ "state": state,+ "verdict": None,+ "reason": "",+ "deleted": False,+ "reports": 0,+ }++ if state in ("QUEUED", "RUNNING"):+ age = entry.get("age_hours")+ if age is not None and age > max_age_hours:+ # Reported as a FAILURE of collection rather than dropped: it has+ # been billing and will never produce a result, and silence leaves+ # the commit reading "not run" forever.+ record["verdict"] = "reaped"+ # "released", not "deleted": the delete happens in a later step+ # that can be refused, and this reason is posted before it runs.+ record["reason"] = (+ f"the kernel was still {state} after {age:.1f}h, past the {max_age_hours}h "+ "ceiling, so it is released for deletion. It was billing accelerator "+ "quota and would not have produced a result"+ )+ if delete:+ record["deleted"] = launch.delete_kernel(slug, deadline = deadline)+ if record["deleted"]:+ record["reason"] = record["reason"].replace(+ "is released for deletion", "was deleted"+ )+ _log(f"reaped {slug} ({state}, {age:.1f}h)")+ return record+ record["verdict"] = "pending"+ record["reason"] = f"still {state}"+ _log(f"pending {slug} ({state}, age {age if age is None else round(age, 2)}h)")+ return record++ if state == "UNKNOWN":+ # Left alone: an unreadable status says nothing, and both actions here+ # are destructive. The next pass asks again.+ record["verdict"] = "pending"+ record["reason"] = "status unreadable this pass"+ return record++ if state == "GONE":+ record["verdict"] = "gone"+ record["reason"] = "the kernel no longer exists; nothing to collect"+ return record++ if state not in launch.TERMINAL_OK | launch.TERMINAL_BAD:+ # Only a KNOWN terminal state may download, judge, post and delete.+ # Kaggle's enum can grow, and judging an unknown state posts a green+ # `infra` for a run still to come and then deletes it.+ record["verdict"] = "pending"+ record["reason"] = f"state {state} is not one this collector judges; asked again next pass"+ _log(f"unrecognised state {state} for {slug}; kept")+ return record++ # Terminal. Evidence FIRST, delete second: a delete before the download+ # turns a finished run into a result nobody can read.+ dest = outdir / slug.rsplit("/", 1)[-1]+ try:+ evidence_deadline = time.time() + EVIDENCE_BUDGET_SEC+ if deadline is not None:+ evidence_deadline = min(evidence_deadline, deadline)+ evidence = launch.fetch_evidence(slug, dest, deadline = evidence_deadline)+ except Exception as exc: # noqa: BLE001+ record["evidence"] = None+ if _gone(exc):+ # Deleted by another pass between our status call and this+ # download: their verdict stands, ours would be an `infra` on top.+ record["verdict"] = "gone"+ record["reason"] = "another collector finished this kernel first"+ _log(f"{slug} was collected by another pass")+ return record+ # `pending` rather than `infra`: `infra` posts green and releases the+ # kernel, losing a real result to one transient failure.+ record["verdict"] = "pending"+ record["reason"] = (+ f"the kernel finished but its evidence would not download this pass "+ f"({type(exc).__name__}); kept for the next one"+ )+ _log(f"could not collect {slug}: {type(exc).__name__}")+ return record++ record["evidence"] = evidence+ if evidence.get("truncated"):+ # fetch_evidence flags a spent budget instead of raising, so same+ # answer as above: judging a short set reads a run that lost half its+ # notebooks as whatever the surviving half says.+ record["verdict"] = "pending"+ record["reason"] = (+ "the evidence download was incomplete this pass; the kernel is kept for the next one"+ )+ _log(f"incomplete evidence for {slug}; left for the next pass")+ return record++ try:+ reports = launch.extract_reports(dest)+ except Exception as exc: # noqa: BLE001+ # Evidence that downloaded but cannot be read will not read better next+ # pass, and raising here wedges every later one. Nothing was learned, so+ # `infra`, and the kernel is released.+ reports = []+ record["report_error"] = f"{type(exc).__name__}: {exc}"[:200]+ _log(f"unreadable report in {slug}: {type(exc).__name__}")+ record["reports"] = len(reports)+ expect = expected_reports(dest, expect, record.get("kind") or "")+ record["expected"] = expect+ if expect is None:+ # Plan unknown: a failure is still a failure, but completeness cannot+ # be shown, so no pass is claimed.+ verdict, reason = verdict_of(reports, len(reports))+ if verdict == "pass":+ verdict, reason = (+ "infra",+ (+ f"{len(reports)} payload report(s) read, but the kernel predates the "+ "record of how many it was built to produce, so completeness cannot be "+ "judged and no pass is claimed"+ ),+ )+ else:+ verdict, reason = verdict_of(reports, expect)+ if record.get("report_error") and verdict == "infra":+ reason = (+ f"the evidence downloaded but its report could not be read ({record['report_error']})"+ )+ if state in launch.TERMINAL_BAD and verdict == "infra":+ reason = (+ f"the kernel ended {state} without reporting; the session died rather than the code"+ )+ record["verdict"] = verdict+ record["reason"] = reason+ if delete:+ record["deleted"] = launch.delete_kernel(slug, deadline = deadline)+ if not record["deleted"]:+ _log(f"could not delete {slug}; it may keep billing")+ _log(f"collected {slug}: {verdict} ({reason})")+ return record+++def statuses_from(records: list[dict], target_url: str = "") -> list[dict]:+ """The commit statuses to post, one per collected kernel that names a commit.++ THIS is the check that means "the GPU tests passed": the dispatching job+ succeeds by dispatching, so branch protection pointing at it would require+ only that Kaggle accepted a push. A legacy slug carries no sha to attribute+ to and yields nothing.+ """+ out: dict[tuple[str, str], dict] = {}+ for record in records:+ if record.get("verdict") in (None, "pending", "gone"):+ continue+ sha = record.get("sha")+ kind = record.get("kind")+ if not sha or kind not in STATUS_CONTEXTS:+ continue+ context = STATUS_CONTEXTS[kind]+ state = VERDICT_STATE.get(record["verdict"], "success")+ # One line: GitHub shows the description on one line and caps it at 140.+ description = " ".join(f"{record['verdict']}: {record['reason']}".split())[:140]+ status = {+ "sha": sha,+ "context": context,+ "state": state,+ "description": description,+ "target_url": target_url,+ "slug": record["slug"],+ "slugs": [record["slug"]],+ }+ # Two kernels for one commit under one context (slots 1 and 2) must not+ # race to post last: a failure wins whichever kernel found it, and both+ # are named so the delete step releases both. Keyed on the sha as the+ # slug carries it; only the commits API can say whether an 8 and a 12+ # character slug name one commit, so post_statuses merges after+ # resolving.+ key = (sha, context)+ prior = out.get(key)+ if prior is None:+ out[key] = status+ continue+ prior["slugs"].append(record["slug"])+ if state == "failure" and prior["state"] != "failure":+ status["slugs"] = prior["slugs"]+ out[key] = status+ return list(out.values())+++# Verdicts whose kernel is finished with once the status is delivered. `pending`+# and `gone` own nothing to delete; `None` never got as far as a verdict.+DELETABLE = {"pass", "fail", "partial", "infra", "reaped"}++# Ceiling on the release phase. delete_kernel allows three 180-second attempts+# per kernel, so an unbounded release after a full BUDGET_SEC collection could+# outlive the job. Leftovers are released by the next pass.+RELEASE_BUDGET_SEC = 600+++def delete_collected(result_path: Path, posted_path: Path | None) -> int:+ """Release the kernels a ``--no-delete`` pass judged, now that their+ statuses are posted.++ ``posted.json`` names the slugs delivered, refused, and whose commit is+ gone. A refused post KEEPS its kernel, the only retry there is. A gone+ commit releases it; holding it only bills.+ """+ data = json.loads(result_path.read_text(encoding = "utf-8"))+ posted = (+ json.loads(posted_path.read_text(encoding = "utf-8"))+ if posted_path and posted_path.exists()+ else {}+ )+ # Refused and rejected alike: nothing reached GitHub, so the kernel stays.+ failed = set(posted.get("failed") or []) | set(posted.get("invalid") or [])+ if posted_path and not posted_path.exists():+ # The poster never ran. Keep every kernel that had something to post,+ # or their verdicts are lost.+ failed = {s["slug"] for s in data.get("statuses", [])} | {+ slug for s in data.get("statuses", []) for slug in s.get("slugs", [])+ }+ outcome = {"deleted": [], "kept": [], "failed_delete": []}+ deadline = time.time() + RELEASE_BUDGET_SEC+ for record in data.get("kernels") or []:+ slug = record.get("slug")+ if not slug or record.get("verdict") not in DELETABLE:+ continue+ if record.get("deleted"):+ continue+ if slug in failed:+ outcome["kept"].append(slug)+ _log(f"kept {slug}: its status did not post; the next pass retries")+ continue+ if time.time() >= deadline:+ outcome["kept"].append(slug)+ _log(f"kept {slug}: release budget spent; the next pass releases it")+ continue+ if launch.delete_kernel(slug, deadline = deadline):+ outcome["deleted"].append(slug)+ _log(f"deleted {slug}")+ else:+ outcome["failed_delete"].append(slug)+ print(f"::warning title=Kaggle kernel not deleted::{slug} may keep billing")+ result_path.with_name("delete_result.json").write_text(+ json.dumps(outcome, indent = 2), encoding = "utf-8"+ )+ return 0+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--outdir", required = True, help = "where evidence is downloaded")+ ap.add_argument(+ "--expect",+ type = int,+ default = 1,+ help = "payload reports a collected kernel should carry",+ )+ ap.add_argument("--max-age-hours", type = float, default = DEFAULT_MAX_AGE_HOURS)+ ap.add_argument(+ "--sha",+ default = "",+ help = "when given, report whether a kernel for THIS commit is already in "+ "flight, so the caller can skip dispatching a second one",+ )+ ap.add_argument(+ "--kind",+ default = "",+ choices = ("", *launch.KIND_CODES),+ help = "narrow --sha to one workflow's kernels",+ )+ ap.add_argument(+ "--slot",+ default = "1",+ help = "narrow --sha to one session slot's kernels: slot 2 runs beside slot 1 on "+ "the same commit by design, and only its own retry is a duplicate",+ )+ ap.add_argument("--target-url", default = "", help = "run URL to attach to each status")+ ap.add_argument(+ "--no-delete",+ action = "store_true",+ help = "collect and report without deleting. The workflows always pass "+ "this and delete afterwards with --delete-collected, once the statuses "+ "are posted; a kernel left up after that keeps billing",+ )+ ap.add_argument(+ "--require-auth",+ action = "store_true",+ help = "fail rather than skip when Kaggle authentication fails. The "+ "scheduled reaper passes this: it runs with the repository's own "+ "secrets, so an auth failure there is a broken token, not a fork",+ )+ ap.add_argument(+ "--delete-collected",+ default = "",+ metavar = "COLLECT_RESULT_JSON",+ help = "delete the kernels a previous --no-delete pass collected, except "+ "those whose status did not post (see --posted). Nothing else runs",+ )+ ap.add_argument(+ "--posted",+ default = "",+ metavar = "POSTED_JSON",+ help = "post_statuses.py's record of which statuses were delivered",+ )+ args = ap.parse_args()++ if args.delete_collected:+ return delete_collected(+ Path(args.delete_collected), Path(args.posted) if args.posted else None+ )++ socket.setdefaulttimeout(SOCKET_TIMEOUT_SEC)+ outdir = Path(args.outdir)+ outdir.mkdir(parents = True, exist_ok = True)+ result: dict = {"owner": None, "kernels": [], "statuses": [], "in_flight_for_sha": False}++ def finish(code: int = 0) -> int:+ (outdir / "collect_result.json").write_text(json.dumps(result, indent = 2), encoding = "utf-8")+ _out("in_flight", "true" if result["in_flight_for_sha"] else "false")+ _out(+ "collected",+ str(+ sum(+ 1+ for k in result["kernels"]+ if k.get("verdict") not in (None, "pending", "gone")+ )+ ),+ )+ _out("pending", str(sum(1 for k in result["kernels"] if k.get("verdict") == "pending")))+ return code++ if args.require_auth and not os.environ.get("KAGGLE_API_TOKEN"):+ # Not configured is not broken: the collector's matrix is static, so a+ # repository with one Kaggle account leaves the second secret empty.+ # Warn and pass. A token that is present and refused is still red.+ print(+ "::warning title=Kaggle account not configured::KAGGLE_API_TOKEN is empty in "+ "this job, so nothing was collected for this account. A repository with one "+ "Kaggle account leaves the second secret unset; if this account is meant to "+ "be configured, set its secret."+ )+ _log("no token in this job; nothing collected")+ return finish()++ try:+ api = launch._api()+ except BaseException as exc: # noqa: BLE001+ if isinstance(exc, KeyboardInterrupt):+ raise+ if args.require_auth:+ # The reaper runs on the repository's own token, so this is an+ # expired credential. A green empty pass would hide it while kernels+ # bill to their ceiling and commits stay pending.+ _log(f"kaggle auth failed ({type(exc).__name__}) and this pass requires it")+ print(+ "::error title=Kaggle authentication failed::the collector could not "+ f"authenticate ({type(exc).__name__}). Nothing was collected or reaped; "+ "dispatched kernels keep billing and their commits stay pending until "+ "the token is repaired."+ )+ return finish(1)+ # A skip, not a failure, for the same reason gate.py skips: on a fork+ # pull request the secret is withheld and there is nothing to fix.+ _log(f"kaggle auth failed ({type(exc).__name__}); nothing collected")+ return finish()++ owner = launch.username_of(api)+ result["owner"] = owner+ _log(f"authenticated as {owner}")++ # The budget starts BEFORE the listing: five slow pages under the socket+ # timeout are minutes, and a deadline started after them is that much later+ # than the job timeout was sized for.+ deadline = time.time() + BUDGET_SEC+ ours = find_ours(api, max_age_hours = args.max_age_hours, deadline = deadline)+ _log(f"{len(ours)} kernel(s) of ours on this account")++ for entry in ours:+ if time.time() >= deadline:+ # Safe by construction: nothing was deleted that was not first+ # collected. Logged so a partial pass is not read as an empty one.+ _log("collection budget spent; the rest is left for the next pass")+ break+ result["kernels"].append(+ collect_one(+ api,+ entry,+ outdir,+ args.expect,+ args.max_age_hours,+ delete = not args.no_delete,+ deadline = deadline,+ )+ )++ result["statuses"] = statuses_from(result["kernels"], args.target_url)++ if args.sha:+ want = args.sha.strip().lower()+ # Prefix match either way: SLUG_SHA_LEN characters now, eight before,+ # and both can be on the account at once.+ result["in_flight_for_sha"] = any(+ k.get("sha")+ and (want.startswith(k["sha"]) or k["sha"].startswith(want))+ and k.get("verdict") == "pending"+ and (not args.kind or k.get("kind") == args.kind)+ and k.get("slot", "1") == str(args.slot or "1").strip()+ for k in result["kernels"]+ )+ _log(f"in flight for {want}: {result['in_flight_for_sha']}")++ return finish()+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/scripts/kaggle_t4_ci/gate.py b/.github/scripts/kaggle_t4_ci/gate.pyindex efe1033dedb..d1c11f6c1ef 100644--- a/.github/scripts/kaggle_t4_ci/gate.py+++ b/.github/scripts/kaggle_t4_ci/gate.py@@ -358,22 +358,56 @@ def scaled_reserve(reserve_hours: float, total_hours: float, basis_hours: float)return round(reserve_hours * (total_hours / basis_hours), 3)-def weighted_pick(run_id: str, weights: dict[str, float]) -> tuple[str, float]:- """Deterministic weighted choice of account, keyed on the run id ALONE.+def in_flight_for_commit(+ own_busy: list[str],+ head_sha: str,+ kind: str,+ slot: str = "1",+) -> str | None:+ """The ref of a busy kernel of ours already running THIS commit, kind and+ slot, or None.++ The slot is part of the identity: a slot-1 kernel must not stand a slot-2+ dispatch down, while a retry of the slot-2 run itself must be.++ Asked of every account surveyed, not only the one picked: a handover lands+ a retry on the other account, whose GPU job collects with only its own+ token and would see nothing in flight.+ """+ sha = (head_sha or "").strip().lower()+ slot = str(slot or "1").strip()+ if not sha or not kind:+ return None+ import launch # noqa: PLC0415 (sibling script, loaded lazily so the gate imports alone)++ for entry in own_busy:+ ref = entry.split(" (", 1)[0]+ parsed = launch.parse_slug(ref)+ if not parsed or parsed.get("kind") != kind or not parsed.get("sha"):+ continue+ if parsed.get("slot", "1") != slot:+ continue+ if sha.startswith(parsed["sha"]) or parsed["sha"].startswith(sha):+ return ref+ return None- Not the attempt, for the same reason `sampled_in` excludes it and a sharper- one: a re-run of a run whose kernels are still in flight must return to the- SAME account, or the second attempt pushes to an account the first one is- not watching and the first one's kernels are reaped by nobody.++def weighted_pick(key: str, weights: dict[str, float]) -> tuple[str, float]:+ """Deterministic weighted choice of account, keyed on the COMMIT under test+ (the run id when no commit is known).++ The commit rather than the run or the attempt: every run of one commit+ must land on the account already holding that commit's kernel, or the+ collector on the other account sees nothing in flight, dispatches a+ duplicate, and reaps nothing.Salted differently from `sampled_in` so the two draws off one run id are- independent -- without the salt, whether a run is sampled in would correlate- with which account it lands on, and one account would quietly get a- different SHARE of the forced runs than of the sampled ones.+ independent; without the salt one account would get a different SHARE of+ the forced runs than of the sampled ones."""ids = sorted(weights)total = sum(max(0.0, weights[i]) for i in ids)- digest = hashlib.sha256(("account:" + run_id).encode("utf-8")).hexdigest()+ digest = hashlib.sha256(("account:" + key).encode("utf-8")).hexdigest()draw = (int(digest[:8], 16) % 1_000_000) / 1_000_000.0if not ids or total <= 0:return (ids[0] if ids else ""), draw@@ -664,6 +698,25 @@ def main() -> int:)ap.add_argument("--run-id", default = os.environ.get("GITHUB_RUN_ID", "0"))ap.add_argument("--run-attempt", default = os.environ.get("GITHUB_RUN_ATTEMPT", "1"))+ ap.add_argument(+ "--head-sha",+ default = "",+ help = "the commit under test; the account draw is keyed on it so every run of one "+ "commit lands on the account holding its kernel",+ )+ ap.add_argument(+ "--kind",+ default = "",+ help = "notebook or studio: with --head-sha, a kernel of this kind already running "+ "this commit on ANY account stands the run down instead of dispatching a duplicate",+ )+ ap.add_argument(+ "--slot",+ default = "1",+ help = "the workflow's session slot input. Slot 2 is an opt-in SECOND session on "+ "the same commit beside slot 1: only a kernel already running this commit in "+ "the SAME slot stands the run down",+ )ap.add_argument("--force", default = "false", help = "workflow_dispatch force input")ap.add_argument("--labels", default = "", help = "comma or newline separated PR labels")ap.add_argument("--label-name", default = "kaggle-t4-ci")@@ -857,12 +910,11 @@ def main() -> int:print("[gate] accounts " + json.dumps(list(probes.values())), flush = True)_out("accounts", json.dumps(list(probes.values())))- # An account whose quota could not be read has no weight, because a weight- # is what its plan says and we did not get to hear it. It stays a CANDIDATE- # -- the order below still reaches it -- so an unreadable answer costs the- # account its share of the traffic and not its place in the queue.+ # An unreadable quota means no weight but still a CANDIDATE: it costs the+ # account its share of the traffic, not its place in the queue.weights = {i: p["total_hours"] for i, p in probes.items() if p.get("total_hours")}- sampled_account, account_draw = weighted_pick(args.run_id, weights)+ account_key = (args.head_sha or "").strip().lower() or str(args.run_id)+ sampled_account, account_draw = weighted_pick(account_key, weights)if weights:share = max(0.0, weights.get(sampled_account, 0.0)) / sum(weights.values())print(@@ -878,11 +930,45 @@ def main() -> int:flush = True,)- # The sampled account first, then the rest in declaration order. Only the- # account actually being considered pays for a survey, which is the- # expensive call here.+ # Sampled account first, then declaration order. Only an account actually+ # considered pays for a survey, the expensive call here.order = [sampled_account] + [i for i in probes if i != sampled_account]handovers: list[str] = []+ surveys: dict[str, dict] = {}+ # ONE survey budget for the whole gate: two bounded surveys back to back+ # outlive the job timeout, and a runner killed mid-question is a red rather+ # than the soft stand-down an incomplete survey gives.+ survey_deadline = time.monotonic() + SURVEY_BUDGET_SEC++ def _survey(account_id: str) -> dict:+ if account_id not in surveys:+ surveys[account_id] = survey_kernels(+ clients[account_id],+ budget_sec = max(0.0, survey_deadline - time.monotonic()),+ )+ return surveys[account_id]++ # Ask EVERY usable account about this commit before choosing one: an+ # earlier run may have handed the commit to the other account, and a retry+ # finding the preferred one free again would never look at the kernel still+ # running. Capacity for a NEW launch is beside the point here.+ slot = str(args.slot or "1").strip()+ if args.head_sha and args.kind:+ for account_id in order:+ if not account_id or clients.get(account_id) is None:+ continue+ try:+ survey = _survey(account_id)+ except Exception: # noqa: BLE001+ continue # reported below, where the account is considered+ already = in_flight_for_commit(survey["own"], args.head_sha, args.kind, slot)+ if already:+ return _decide(+ False,+ f"a {args.kind} kernel for this commit (slot {slot}) is already running "+ f"on account {account_id} ({already}); its result arrives as the commit "+ "status, so nothing is dispatched",+ )for account_id in order:if not account_id:@@ -892,9 +978,8 @@ def main() -> int:handovers.append(f"account {account_id} {record['outcome']}")continue- api = clients[account_id]try:- survey = survey_kernels(api)+ survey = _survey(account_id)except Exception as exc: # noqa: BLE001record["outcome"] = "capacity_unreadable"record["error"] = type(exc).__name__diff --git a/.github/scripts/kaggle_t4_ci/launch.py b/.github/scripts/kaggle_t4_ci/launch.pyindex 9ce3eac7e6e..7f79a29270d 100644--- a/.github/scripts/kaggle_t4_ci/launch.py+++ b/.github/scripts/kaggle_t4_ci/launch.py@@ -86,6 +86,88 @@ TERMINAL_BAD = {"ERROR", "CANCEL_REQUESTED", "CANCEL_ACKNOWLEDGED"}_STATUS_RE = re.compile(r"KernelWorkerStatus\.(?P<status>[A-Z_]+)")+# The slug is the ONLY record that survives the runner.+#+# Dispatch mode exits minutes after the push and a DIFFERENT job collects,+# possibly days later, with no artifact or database between them. Kaggle's+# kernel listing is that database, and a kernel carries exactly one field we+# control: its name.+#+# unsloth-t4-ci-n1a2b3c4d5e6f-9f8e+# ^^^^^^^^^^^^^^^^^^+# ||||||||||||| uniqueness, so two runs on one commit do not+# |||||||||||| collide (a re-run, or notebook slot 1 and 2)+# |kind which workflow to report against+# sha12 which commit the result belongs to. Twelve,+# not eight: two commits sharing eight hex+# characters give indistinguishable kernels, and+# the commits API then 422s the prefix on every+# pass, so the status stays pending forever.+#+# `OWN_KERNEL_PREFIX` in gate.py is the first 14 characters and MUST keep+# matching: it is how the survey tells our kernels from a human's, and one it+# cannot recognise is not counted against Kaggle's 2-session cap.+SLUG_PREFIX = "unsloth-t4-ci-"++# One character each, so the slug survives `_slugify` and stays under Kaggle's+# length limit. A wire format, not an enum: a collector reads it back days later.+KIND_CODES = {"notebook": "n", "studio": "s"}+CODE_KINDS = {v: k for k, v in KIND_CODES.items()}++# Legacy slugs (`unsloth-t4-ci-<8 hex>`) are still OURS and must still be+# reaped, but carry no commit to report against. The alternation reads 8 to 40+# hex rather than making the collector guess from length.+SLUG_SHA_LEN = 12+# The `slot` input, when it is not 1, sits between the dash and the uid:+# `-2<uid4>`. Slot 2 is a deliberate SECOND session on the same commit, so the+# duplicate check has to say "this commit, this kind, THIS slot". Absent means+# slot 1, so every slug pushed before the slot existed still reads.+CI_SLUG_RE = re.compile(+ r"^(?:(?P<owner>[^/]+)/)?"+ + re.escape(SLUG_PREFIX)+ + r"(?:(?P<kind>[a-z])(?P<sha>[0-9a-f]{8,40})-(?P<slot>[2-9])?(?P<uid>[0-9a-f]{4})"+ r"|(?P<legacy>[0-9a-f]{8}))$"+)+++def parse_slug(slug: str) -> dict | None:+ """What a kernel name says about itself, or None if it is not ours.++ None for anything unrecognised is the collector's safety property: it+ enumerates a shared ACCOUNT and deletes what it collects, so a collector+ that guessed would eventually delete someone's work.+ """+ match = CI_SLUG_RE.match(slug or "")+ if match is None:+ return None+ return {+ "owner": match.group("owner"),+ "kind": CODE_KINDS.get(match.group("kind") or "", None),+ "sha": match.group("sha"),+ "slot": match.group("slot") or "1",+ "legacy": match.group("legacy") is not None,+ }+++def slug_name(+ kind: str = "",+ commit_sha: str = "",+ slot: str = "1",+) -> str:+ """The name for a kernel about to be pushed.++ Falls back to the legacy random form when no commit is named, as a local+ probe or an older workflow does. A dispatched run always names one.+ """+ code = KIND_CODES.get(kind, "")+ sha = (commit_sha or "").strip().lower()[:SLUG_SHA_LEN]+ slot = str(slot or "1").strip()+ mark = slot if len(slot) == 1 and slot in "23456789" else ""+ if code and len(sha) == SLUG_SHA_LEN and all(c in "0123456789abcdef" for c in sha):+ return f"{SLUG_PREFIX}{code}{sha}-{mark}{uuid.uuid4().hex[:4]}"+ return f"{SLUG_PREFIX}{uuid.uuid4().hex[:8]}"++PUSH_ATTEMPTS = 4PUSH_BACKOFF_SEC = 45@@ -181,7 +263,11 @@ OUTPUT_PAGE_LIMIT = 20READ_CHUNK_BYTES = 1 << 16-def worst_case_seconds(max_wait: int, kernels: int) -> int:+def worst_case_seconds(+ max_wait: int,+ kernels: int,+ dispatch: bool = False,+) -> int:"""Wall clock ONE invocation of this launcher can take, from its constants.Every phase that can keep a pushed kernel UP is in it, because a kernel@@ -209,6 +295,11 @@ def worst_case_seconds(max_wait: int, kernels: int) -> int:+ sum(PUSH_BACKOFF_SEC * 2**i for i in range(PUSH_ATTEMPTS - 1))+ (PUSH_ATTEMPTS - 1) * one_delete)+ if dispatch:+ # Dispatch stops after the pushes, so the phases below are unreachable+ # and counting them would make the pre-push guard demand ~2h for a job+ # that exits in five. The retry _discard()s stay in: those run in push().+ return kernels * per_pushreturn (max(kernels * per_push, max_wait)+ EVIDENCE_BUDGET_SEC@@ -499,6 +590,9 @@ def push(kernel_timeout_sec: int,accelerator: str = "NvidiaTeslaT4",attempted: list[str] | None = None,+ kind: str = "",+ commit_sha: str = "",+ slot: str = "1",) -> dict:"""Push as a fresh private kernel. Every attempt gets its own slug.@@ -527,7 +621,6 @@ def push(``MemoryError``) would otherwise take the slug of a push Kaggle may haveACCEPTED with it, leaving a kernel nothing will delete. See main()."""- base = _slugify("unsloth t4 ci")[:32]attempted = [] if attempted is None else attempteddef _discard(slug: str) -> None:@@ -545,22 +638,23 @@ def push(for attempt in range(PUSH_ATTEMPTS):if attempted:_discard(attempted[-1])- slug_name = f"{base}-{uuid.uuid4().hex[:8]}"+ name = slug_name(kind, commit_sha, slot)# The slug derives from the TITLE, not the metadata id: a mismatch- # files the kernel at an unexpected address and every later- # status/output call 403s, so assert the round trip.- title = slug_name.replace("-", " ")- assert _slugify(title) == slug_name, f"title {title!r} slugifies to {_slugify(title)!r}"- attempted.append(f"{user}/{slug_name}")+ # files the kernel at an unexpected address, every later+ # status/output call 403s and no collector can attribute it. So+ # assert the round trip.+ title = name.replace("-", " ")+ assert _slugify(title) == name, f"title {title!r} slugifies to {_slugify(title)!r}"+ attempted.append(f"{user}/{name}")for stale in workdir.glob("*.ipynb"):stale.unlink()- code_file = workdir / f"{slug_name}.ipynb"+ code_file = workdir / f"{name}.ipynb"shutil.copy(notebook, code_file)(workdir / "kernel-metadata.json").write_text(json.dumps({- "id": f"{user}/{slug_name}",+ "id": f"{user}/{name}","title": title,"code_file": code_file.name,"language": "python",@@ -650,9 +744,13 @@ def _already_gone(text: str) -> bool:return any(marker in text.lower() for marker in GONE_MARKERS)-def delete_kernel(slug: str) -> bool:+def delete_kernel(slug: str, deadline: float | None = None) -> bool:"""Delete one kernel, and answer whether Kaggle actually deleted it.+ ``deadline`` (absolute ``time.time()``) bounds the whole call: attempts are+ clamped to what is left and none starts past it, so a release phase sharing+ one deadline ends when it says it will rather than three attempts after.+``subprocess.run`` does not raise on a nonzero exit, so the caller used torecord every slug as released whatever came back: a refused delete, anexpired token, or the case that made this visible, where the pinned client@@ -686,12 +784,21 @@ def delete_kernel(slug: str) -> bool:this refuses needs a human, which is what the caller's warning is for."""for attempt in range(DELETE_ATTEMPTS):+ timeout = DELETE_SUBPROCESS_TIMEOUT_SEC+ if deadline is not None:+ left = deadline - time.time()+ if left <= 0:+ _log(+ f"delete {slug}: budget spent before attempt {attempt + 1}; left for the next pass"+ )+ return False+ timeout = min(timeout, left)try:proc = subprocess.run(["kaggle", "kernels", "delete", slug, "-y"],capture_output = True,text = True,- timeout = DELETE_SUBPROCESS_TIMEOUT_SEC,+ timeout = timeout,)except Exception as exc: # noqa: BLE001_log(f"delete {slug} did not run: {type(exc).__name__}")@@ -941,6 +1048,10 @@ def fetch_evidence(continueurl = entry.get("url") or entry.get("urlNullable")if not url:+ # Listed but not downloadable is missing evidence: judged without+ # it, a lost failing payload reads as whatever survived.+ _log(f"{name} was listed without a download URL; evidence incomplete")+ truncated = Truecontinuecall_timeout = _time_left(deadline, timeout)if call_timeout is None:@@ -1019,6 +1130,10 @@ def extract_reports(outdir: Path) -> list[dict]:parsed = json.loads(blob)except json.JSONDecodeError:continue+ # A report is an object; `.get` on anything else that parses raises+ # out of the collector and wedges every later pass on this evidence.+ if not isinstance(parsed, dict):+ continuekey = f"{parsed.get('label')}|{parsed.get('model')}"if key in seen:continue@@ -1032,11 +1147,23 @@ def extract_reports(outdir: Path) -> list[dict]:nb = json.loads(nb_path.read_text(encoding = "utf-8", errors = "replace"))except Exception: # noqa: BLE001continue- for cell in nb.get("cells", []):- for output in cell.get("outputs", []):+ # Valid JSON is not necessarily a notebook, and one malformed file must+ # not take its neighbours' reports with it: an exception here used to+ # reach the collector, which then judged the kernel infra, posted+ # success and released it, hiding a real failure behind a mangled file.+ if not isinstance(nb, dict):+ continue+ for cell in nb.get("cells") or []:+ if not isinstance(cell, dict):+ continue+ for output in cell.get("outputs") or []:+ if not isinstance(output, dict):+ continuetext = output.get("text") or ""if isinstance(text, list):- text = "".join(text)+ text = "".join(str(t) for t in text)+ if not isinstance(text, str):+ continue_consume(text)for log_path in sorted(outdir.rglob("kernel.log")):raw = log_path.read_text(encoding = "utf-8", errors = "replace")@@ -1152,6 +1279,33 @@ def main() -> int:ap.add_argument("--keep-kernel", action = "store_true", help = "do not delete the kernel after collecting")+ ap.add_argument(+ "--dispatch",+ action = "store_true",+ help = "push and EXIT, without waiting, collecting or deleting. The kernel "+ "is collected later by collect.py, which finds it by its slug",+ )+ ap.add_argument(+ "--commit-sha",+ default = "",+ help = "the commit under test. Written into the slug, which is the only "+ "record a later collector has of what the result is about",+ )+ ap.add_argument(+ "--kind",+ default = "",+ choices = ("", *KIND_CODES),+ help = "which workflow is dispatching, so the collector knows which "+ "commit status context to report the result under",+ )+ ap.add_argument(+ "--slot",+ default = "1",+ choices = tuple("123456789"),+ help = "the workflow's session slot. Written into the slug when it is not 1, "+ "so a retry of a slot-2 run is recognised as the same session and a slot-2 "+ "dispatch beside slot 1 is not",+ )ap.add_argument("--deadline-epoch",type = int,@@ -1161,6 +1315,29 @@ def main() -> int:)args = ap.parse_args()+ # A dispatched kernel is left running, which is exactly what --keep-kernel+ # already means down to the registry bookkeeping. Reusing the flag rather+ # than giving release() a second way not to delete: two conditions guarding+ # one deletion is how one ends up wrong, and this one bills GPU quota.+ if args.dispatch:+ args.keep_kernel = True+ # Refused rather than defaulted: a slug with no commit runs, costs quota+ # and reports to nobody. A usage error at the only moment it is cheap.+ if not args.commit_sha or not args.kind:+ ap.error(+ "--dispatch requires --commit-sha and --kind: without both, the "+ "kernel runs and no collector can attribute its result"+ )+ # A commit, not a ref: `slug_name` carries only hex and falls back to+ # the legacy unattributable form, so a branch or tag here bills and+ # reports to nobody. The workflows resolve refs; this checks they did.+ if not re.fullmatch(r"[0-9a-fA-F]{12,40}", args.commit_sha.strip()):+ ap.error(+ f"--commit-sha must be a hex commit id (12 to 40 characters), got "+ f"{args.commit_sha!r}: a ref cannot be written into the slug, so "+ "the kernel's result could never be attributed to a commit"+ )+# Before the first network call, and globally: the Kaggle client has no# per-call timeout of its own. See SOCKET_TIMEOUT_SEC.socket.setdefaulttimeout(SOCKET_TIMEOUT_SEC)@@ -1248,7 +1425,7 @@ def main() -> int:"""if not args.deadline_epoch:return True- need = worst_case_seconds(args.max_wait, len(args.notebook))+ need = worst_case_seconds(args.max_wait, len(args.notebook), dispatch = args.dispatch)left = int(args.deadline_epoch - time.time())if left >= need:_log(f"{left}s left of the job deadline, worst case {need}s ({after})")@@ -1398,6 +1575,9 @@ def main() -> int:args.user,args.kernel_timeout_sec,attempted = entry["attempted"],+ kind = args.kind,+ commit_sha = args.commit_sha,+ slot = args.slot,)entry["slug"] = pushed.get("slug")entry["push_error"] = (@@ -1416,6 +1596,22 @@ def main() -> int:# single-kernel shape.result["slug"] = live[0]["slug"]+ # DISPATCH MODE ENDS HERE. `dispatched` is deliberately NOT `pass`: all+ # this job proved is that Kaggle accepted a push. The real verdict comes+ # later from collect.py as a commit status on the sha in the slug.+ if args.dispatch:+ result["verdict"] = "dispatched"+ result["dispatched"] = [{"slug": k["slug"], "notebook": k["notebook"]} for k in live]+ result["commit_sha"] = args.commit_sha+ result["kind"] = args.kind+ result["reason"] = (+ f"{len(live)} kernel(s) pushed and left running: "+ + ", ".join(k["slug"] for k in live)+ + f". The result for {args.commit_sha[:8]} will be posted as a commit "+ "status by the collector, not by this job"+ )+ return finish()+# The deadline above is shared, not one per kernel: they run# concurrently, so consuming the ceiling per kernel would let a# two-kernel run wait twice its own stated bound.diff --git a/.github/scripts/kaggle_t4_ci/post_statuses.py b/.github/scripts/kaggle_t4_ci/post_statuses.pynew file mode 100644index 00000000000..7fc2bcd5690--- /dev/null+++ b/.github/scripts/kaggle_t4_ci/post_statuses.py@@ -0,0 +1,191 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Post the commit statuses collect.py decided on, and say which ones landed.++The only step holding a GitHub credential, and it holds no Kaggle one:+collect.py judges, this posts, neither can leak the other's token. A script+rather than a shell loop in three workflows so the record stays JSON end to end+and each value reaches `gh` as an argument, never through a shell.++THE SHA HAS TO BE EXPANDED FIRST. The slug carries an abbreviation (a full sha+does not fit Kaggle's slug limit) and the statuses API refuses one, measured+against a real repository:++ POST /statuses/2ecb19df+ 422 "Sha must be a valid hex object ID"++A sha that cannot be resolved (force-pushed away) is recorded as `unresolved`+so the kernel is released rather than retried forever.++``posted.json`` feeds collect.py --delete-collected: a kernel whose status did+not post is KEPT on Kaggle for the next pass. That ordering (post, then delete)+is why this is a separate step.+"""++from __future__ import annotations++import argparse+import json+import os+import re+import subprocess+import sys+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))++from collect import STATUS_CONTEXTS # noqa: E402++STATES = ("error", "failure", "pending", "success")+FULL_SHA = re.compile(r"^[0-9a-f]{40}$")+++def _gh(args: list[str]) -> tuple[int, str, str]:+ proc = subprocess.run(["gh", *args], capture_output = True, text = True)+ return proc.returncode, (proc.stdout or "").strip(), (proc.stderr or "").strip()+++# What GitHub says when the commit is genuinely not there, measured:+#+# gh api repos/unslothai/unsloth/commits/deadbeef00+# {"message":"No commit found for SHA: deadbeef00", ..., "status":"422"}+#+# Only that message releases the kernel. Status codes do not say it: 404 is+# also an unreadable repository, 422 an ambiguous abbreviation, and reading+# either as "gone" would delete the only copy of the result.+MISSING_MARKERS = ("no commit found",)+++def resolve_sha(repo: str, sha: str) -> tuple[str, str | None]:+ """``("ok", full_sha)``, ``("missing", None)`` when the repository no+ longer has the commit, or ``("error", None)`` when the lookup itself+ failed and nothing is known about the commit."""+ code, out, err = _gh(["api", f"repos/{repo}/commits/{sha}", "-q", ".sha"])+ if code == 0 and FULL_SHA.match(out):+ return "ok", out+ text = f"{out} {err}".lower()+ if any(m in text for m in MISSING_MARKERS):+ return "missing", None+ return "error", None+++def post_one(repo: str, full_sha: str, status: dict) -> bool:+ code, _out, _err = _gh(+ [+ "api",+ f"repos/{repo}/statuses/{full_sha}",+ "-f",+ f"state={status['state']}",+ "-f",+ f"context={status['context']}",+ "-f",+ f"description={status['description']}",+ "-f",+ f"target_url={status.get('target_url') or ''}",+ "--silent",+ ]+ )+ return code == 0+++def valid(status: dict) -> str | None:+ """Why this record must not be posted, or None if it is well formed.++ Checked here because this is the process holding the token: a state outside+ the API's four, or a context not ours, is a record we must not sign.+ """+ if status.get("state") not in STATES:+ return f"state {status.get('state')!r} is not one of {STATES}"+ if status.get("context") not in STATUS_CONTEXTS.values():+ return f"context {status.get('context')!r} is not a Kaggle CI context"+ if not re.fullmatch(r"[0-9a-f]{8,40}", str(status.get("sha") or "")):+ return f"sha {status.get('sha')!r} is not a hex commit id"+ return None+++def merge_by_commit(resolved: list[tuple[str, dict]]) -> list[tuple[str, dict]]:+ """One status per (full sha, context), failure winning, every slug named.++ Merged HERE and not in the collector, because only a resolved sha says+ whether two slugs name one commit: an 8 and a 12 character slug for one+ commit resolve alike, two commits sharing 8 characters do not.+ """+ out: dict[tuple[str, str], dict] = {}+ for full, status in resolved:+ key = (full, status["context"])+ prior = out.get(key)+ if prior is None:+ out[key] = dict(status, slugs = list(status.get("slugs") or [status.get("slug")]))+ continue+ slugs = prior["slugs"] + list(status.get("slugs") or [status.get("slug")])+ if status["state"] == "failure" and prior["state"] != "failure":+ out[key] = dict(status, slugs = slugs)+ else:+ prior["slugs"] = slugs+ return [(k[0], s) for k, s in out.items()]+++def post_all(statuses: list[dict], repo: str) -> dict:+ outcome: dict[str, list[str]] = {"ok": [], "failed": [], "unresolved": [], "invalid": []}+ resolved: list[tuple[str, dict]] = []+ for status in statuses:+ slugs = list(status.get("slugs") or [status.get("slug")])+ why = valid(status)+ if why:+ print(f"::warning title=Malformed status record::{why}; not posted")+ outcome["invalid"].extend(slugs)+ continue+ found, full = resolve_sha(repo, status["sha"])+ if found == "missing":+ print(+ f"::warning title=Could not resolve a collected commit::{status['sha']} is "+ f"no longer a commit in this repository, so its {status['context']} result "+ "cannot be posted. The kernel is released; nothing will ever post for it."+ )+ outcome["unresolved"].extend(slugs)+ continue+ if found != "ok":+ print(+ f"::warning title=Commit lookup failed::could not ask GitHub about "+ f"{status['sha']} this pass; the kernel is kept so the next pass can try again"+ )+ outcome["failed"].extend(slugs)+ continue+ resolved.append((full, status))+ for full, status in merge_by_commit(resolved):+ slugs = status["slugs"]+ print(f"posting {status['context']}={status['state']} for {full}")+ if post_one(repo, full, status):+ outcome["ok"].extend(slugs)+ else:+ print(+ f"::warning title=Could not post a commit status::{status['context']} for "+ f"{full}; the kernel is kept so the next pass can try again"+ )+ outcome["failed"].extend(slugs)+ return outcome+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--result", required = True, help = "collect_result.json from collect.py")+ ap.add_argument("--out", required = True, help = "where to write posted.json")+ ap.add_argument("--repo", default = os.environ.get("GITHUB_REPOSITORY", ""))+ args = ap.parse_args()+ if not args.repo:+ ap.error("--repo (or GITHUB_REPOSITORY) is required")++ data = json.loads(Path(args.result).read_text(encoding = "utf-8"))+ statuses = data.get("statuses") or []+ outcome = post_all(statuses, args.repo)+ Path(args.out).write_text(json.dumps(outcome, indent = 2), encoding = "utf-8")+ if not statuses:+ print("no statuses to post this pass")+ # Red when a verdict could not be delivered: the kernel is kept for the+ # next pass, but a delivery failure must not look like a quiet account.+ return 1 if outcome["failed"] else 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/workflows/kaggle-collect.yml b/.github/workflows/kaggle-collect.ymlnew file mode 100644index 00000000000..a9c491d6d11--- /dev/null+++ b/.github/workflows/kaggle-collect.yml@@ -0,0 +1,214 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+#+# Collect the Kaggle kernels the GPU workflows dispatched and left running.+#+# WHY THIS EXISTS+#+# The GPU workflows used to hold a runner for the whole Kaggle session: 41.5+# and 18.7 minutes, measured on runs 33479481067 and 33486360729, almost all of+# it polling, against a queue measured 400-670 deep.+#+# So those jobs now PUSH AND EXIT (`launch.py --dispatch`), and two independent+# routes collect: the next GPU run on the same account collects before it+# dispatches, and THIS workflow covers a quiet repository on a schedule. The+# schedule is not a nicety; a kernel here has been measured ignoring Kaggle's+# own `-t` timeout for over two hours, billing the whole time.+#+# WHAT IT PUBLISHES+#+# A commit status per collected kernel, `kaggle-t4-notebook` or+# `kaggle-studio-gpu`, against the sha in the kernel's slug. THAT is the check+# meaning "the GPU tests passed"; the dispatching job proves only that Kaggle+# accepted a push. Neither context can be a REQUIRED check while the GPU+# workflows sample and path-filter: a commit the gate skips gets no status, and+# a required context that never arrives blocks the merge.+#+# ACCOUNTS+#+# A static two-element matrix, one job per account, unlike the gate's+# quota-weighted draw: the gate chooses where to SPEND a session, this spends+# none and must look at both every pass. One token per job, never both in one+# step, as in the GPU workflows.+name: Kaggle collect++on:+ schedule:+ # Every 10 minutes, offset off the hour: a few passes per 20-40 minute+ # kernel, without the workflow becoming a queue item itself.+ - cron: '3,13,23,33,43,53 * * * *'+ workflow_dispatch:+ inputs:+ max_age_hours:+ description: 'Delete a still-running kernel older than this many hours'+ type: string+ default: '3'+ no_delete:+ description: 'Collect and report without deleting (inspection only; a kernel left up keeps billing)'+ type: boolean+ default: false++permissions:+ contents: read++# Never two scheduled passes at once. This does NOT serialise against the GPU+# jobs, which collect under their own groups, so two collectors CAN see one+# terminal kernel. Safe by construction in collect.py rather than by a lock: a+# vanished kernel is `gone` and posts nothing, an incomplete download keeps the+# kernel, and a status comes only from complete evidence, so the worst case is+# the same verdict twice. Not cancel-in-progress: a pass midway through a+# download must finish, or the next starts from nothing.+concurrency:+ group: kaggle-collect+ cancel-in-progress: false++jobs:+ collect:+ name: Collect+ runs-on: ubuntu-latest+ # collect.py's BUDGET_SEC (900s) plus RELEASE_BUDGET_SEC (600s) plus setup+ # and posting, so the script's budgets stop a long pass rather than the job+ # being killed mid-download.+ timeout-minutes: 30+ permissions:+ contents: read+ # The point of the workflow: without it the verdict reaches nobody.+ statuses: write+ strategy:+ # One unreachable account must not stop the other, and either failing is+ # still a failure.+ fail-fast: false+ matrix:+ include:+ - account_id: '1'+ secret_name: KAGGLE_API_TOKEN+ - account_id: '2'+ secret_name: KAGGLE_API_TOKEN_2+ steps:+ - name: Harden runner (audit)+ uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a+ with:+ egress-policy: audit++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ # The collector holds the Kaggle token; only the posting step gets+ # GH_TOKEN, and checkout persists the job token into .git by default.+ persist-credentials: false++ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97+ with:+ python-version: '3.12'++ - name: Install the Kaggle client+ # As in the GPU workflows: 1.7.4.5's authenticate() cannot read+ # KAGGLE_API_TOKEN, the only credential this job has, so every pass on+ # it collected nothing.+ run: python -m pip install --quiet 'kaggle==2.2.4'++ - name: Collect this account's kernels+ id: collect+ env:+ # Exactly one account, indexed by name. A ternary would hand over the+ # other account's token whenever this secret is empty, and the+ # collector would delete kernels it is not reporting for.+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}+ # --require-auth: this job holds the repository's own secrets, so an+ # auth failure is a broken token and must be red, not the quiet green a+ # fork gets. An EMPTY token is different: the matrix is static, so a+ # repository with one Kaggle account leaves the second secret unset and+ # that leg warns and passes. --no-delete always; the last step releases+ # the kernels once their statuses posted. The report count comes from+ # each kernel's own evidence, so a five-payload kernel that lost four is+ # judged against five rather than a flat guess.+ run: |+ python .github/scripts/kaggle_t4_ci/collect.py \+ --outdir kaggle_collected \+ --require-auth \+ --no-delete \+ --max-age-hours '${{ inputs.max_age_hours || '3' }}' \+ --target-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"++ # Post FIRST, delete SECOND: a verdict that does not reach GitHub leaves+ # its kernel up for the next pass, and deleting first would make one API+ # failure the permanent loss of a verdict. A script rather than a shell+ # loop so each field reaches `gh` as its own argument.+ - name: Post the commit statuses+ id: post+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ python .github/scripts/kaggle_t4_ci/post_statuses.py \+ --result kaggle_collected/collect_result.json \+ --out kaggle_collected/posted.json++ - name: Release the kernels whose verdicts were delivered+ if: always() && !inputs.no_delete && hashFiles('kaggle_collected/collect_result.json') != ''+ env:+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}+ run: |+ python .github/scripts/kaggle_t4_ci/collect.py \+ --outdir kaggle_collected \+ --delete-collected kaggle_collected/collect_result.json \+ --posted kaggle_collected/posted.json++ # The Studio payload smuggles its screenshots and logs home in its own+ # cell output, and this pass collects most Studio kernels, so it has to+ # put them back. One kernel per call: the bundle is reassembled from+ # numbered chunks and one walk over two would splice them together.+ - name: Unpack the Playwright evidence+ if: always() && hashFiles('kaggle_collected/**/*_output.ipynb') != ''+ continue-on-error: true+ run: |+ for dir in kaggle_collected/*/; do+ [ -d "$dir" ] || continue+ python .github/scripts/kaggle_studio_ci/collect_evidence.py \+ --evidence "$dir" \+ --outdir "studio_evidence/$(basename "$dir")"+ done++ # The posted verdict is one line; the notebooks and screenshots are what a+ # red one is read from. continue-on-error because the statuses are already+ # delivered and an artifact outage is not a collection failure.+ - name: Upload evidence+ if: always() && hashFiles('kaggle_collected/**') != ''+ continue-on-error: true+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: kaggle-collect-evidence-account-${{ matrix.account_id }}+ path: |+ kaggle_collected/**+ studio_evidence/**+ retention-days: 14+ if-no-files-found: ignore++ - name: Summarise+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''+ run: |+ python - <<'PY' >> "$GITHUB_STEP_SUMMARY"+ import json, pathlib+ data = json.loads(pathlib.Path("kaggle_collected/collect_result.json").read_text())+ # Collection runs --no-delete, so `deleted` there is always false; the+ # release step wrote the real outcome beside it.+ release = pathlib.Path("kaggle_collected/delete_result.json")+ released = set(json.loads(release.read_text()).get("deleted") or []) if release.exists() else set()+ print(f"### Kaggle collect: account {data.get('owner') or '?'}\n")+ rows = data.get("kernels") or []+ for k in rows:+ k["deleted"] = bool(k.get("deleted")) or k.get("slug") in released+ if not rows:+ print("Nothing of ours was in flight on this account.\n")+ else:+ print("| kernel | commit | kind | state | verdict | deleted |")+ print("| --- | --- | --- | --- | --- | --- |")+ for k in rows:+ print("| `{}` | `{}` | {} | {} | **{}** | {} |".format(+ k["slug"].rsplit("/", 1)[-1], k.get("sha") or "-",+ k.get("kind") or "-", k.get("state") or "-",+ k.get("verdict") or "-", "yes" if k.get("deleted") else "no"))+ print()+ for k in rows:+ if k.get("reason"):+ print(f"- `{k['slug'].rsplit('/', 1)[-1]}`: {k['reason']}")+ PYdiff --git a/.github/workflows/kaggle-t4-notebook-ci.yml b/.github/workflows/kaggle-t4-notebook-ci.ymlindex 409021d13c8..a8cfd37de04 100644--- a/.github/workflows/kaggle-t4-notebook-ci.yml+++ b/.github/workflows/kaggle-t4-notebook-ci.yml@@ -373,7 +373,10 @@ jobs:# and exits 0); this simply does not start.if: github.event.pull_request.head.repo.fork != trueruns-on: ubuntu-latest- timeout-minutes: 10+ # 15, the worst case with two accounts: each one's authentication, username+ # and quota read at the socket ceiling, one shared survey budget, and the+ # call still in flight when it expires. The suite asserts the arithmetic.+ timeout-minutes: 15outputs:should_run: ${{ steps.decide.outputs.should_run }}reason: ${{ steps.decide.outputs.reason }}@@ -383,6 +386,9 @@ jobs:# NAME, and the GPU job indexes the secrets context with it.account: ${{ steps.decide.outputs.account }}account_user: ${{ steps.decide.outputs.account_user }}+ # The commit under test, resolved once: the GPU job tests THIS and never+ # re-resolves a moving ref.+ head_sha: ${{ steps.ref.outputs.head_sha }}matrix: ${{ steps.decide.outputs.matrix }}steps:- name: Harden runner (audit)@@ -464,6 +470,43 @@ jobs:# than being interpolated into the shell, for the same reason the# stale-approval warning below does: they are free text set by anyone with# write access.+ # THE COMMIT THE GPU JOB WILL TEST, which is not always this workflow's+ # own: a dispatch naming `unsloth_ref` tests that ref. Keyed on github.sha+ # instead, the in-flight check would look for the wrong kernel, and once+ # the default branch moved two runs of one requested ref could be drawn+ # onto different accounts and never see each other. A ref that does not+ # resolve leaves the key EMPTY, so the draw falls back to the run id and+ # the GPU job stands down on the same ref moments later.+ - name: Name the commit under test+ id: ref+ env:+ UNSLOTH_REF: ${{ inputs.unsloth_ref }}+ HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}+ run: |+ if [ -z "$UNSLOTH_REF" ]; then+ echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ if printf '%s' "$UNSLOTH_REF" | grep -Eq '^[0-9a-f]{40}$'; then+ RESOLVED="$UNSLOTH_REF"+ else+ RESOLVED=''+ for attempt in 1 2 3; do+ LS="$(git ls-remote https://github.com/unslothai/unsloth "$UNSLOTH_REF" || true)"+ PEELED="$(printf '%s\n' "$LS" | grep '\^{}$' | head -n1 | cut -f1)"+ if [ -n "$PEELED" ]; then+ RESOLVED="$PEELED"+ else+ RESOLVED="$(printf '%s\n' "$LS" | head -n1 | cut -f1)"+ fi+ if [ -n "$RESOLVED" ]; then break; fi+ echo "ls-remote attempt $attempt resolved nothing"+ sleep 10+ done+ fi+ echo "head_sha=$RESOLVED" >> "$GITHUB_OUTPUT"+ echo "the gate is keyed on unslothai/unsloth@${RESOLVED:-(unresolved)}"+- name: Decideid: decideenv:@@ -479,6 +522,9 @@ jobs:run: |python .github/scripts/kaggle_t4_ci/gate.py \--percent 15 \+ --head-sha '${{ steps.ref.outputs.head_sha }}' \+ --kind notebook \+ --slot '${{ inputs.slot || '1' }}' \--force '${{ inputs.force || (github.event_name == 'schedule' && 'true' || 'false') }}' \--labels "$PR_LABELS" \--label-name kaggle-t4-ci \@@ -493,6 +539,13 @@ jobs:needs: gateif: needs.gate.outputs.should_run == 'true'runs-on: ubuntu-latest+ # `statuses: write` is what makes the dispatch honest: this job no longer+ # waits, so it cannot BE the check that says the GPU tests passed. It posts+ # that verdict against the sha the result belongs to, which need not be this+ # run's; the collection step below finishes earlier commits' kernels.+ permissions:+ contents: read+ statuses: write# A ONE-ELEMENT matrix, whose only job is to carry the account the gate# chose into a context that `secrets[...]` can be indexed with.#@@ -738,31 +791,18 @@ jobs:env:UNSLOTH_REF: ${{ inputs.unsloth_ref }}HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}+ GATE_SHA: ${{ needs.gate.outputs.head_sha }}run: |if [ -z "$UNSLOTH_REF" ]; thenecho "ref=$HEAD_SHA" >> "$GITHUB_OUTPUT"echo "testing unslothai/unsloth@$HEAD_SHA"exit 0fi- if printf '%s' "$UNSLOTH_REF" | grep -Eq '^[0-9a-f]{40}$'; then- RESOLVED="$UNSLOTH_REF"- else- RESOLVED=''- for attempt in 1 2 3; do- LS="$(git ls-remote https://github.com/unslothai/unsloth "$UNSLOTH_REF" || true)"- # An annotated tag lists the tag OBJECT first and the commit it- # points at on the `^{}` line, which is what pip can check out.- PEELED="$(printf '%s\n' "$LS" | grep '\^{}$' | head -n1 | cut -f1)"- if [ -n "$PEELED" ]; then- RESOLVED="$PEELED"- else- RESOLVED="$(printf '%s\n' "$LS" | head -n1 | cut -f1)"- fi- if [ -n "$RESOLVED" ]; then break; fi- echo "ls-remote attempt $attempt resolved nothing"- sleep 10- done- fi+ # RESOLVED ONCE, by the gate job, and reused rather than asked again:+ # a branch or tag can move while this job queues, and a second answer+ # could name a commit the gate never keyed the draw or the in-flight+ # check on, whose own run may already be out on the other account.+ RESOLVED="$GATE_SHA"# Every branch above ends here, including the 40-character one. That# branch used to be its own exit: a SHA is immutable, so it was# accepted on shape alone and nothing asked whether unslothai/unsloth@@ -1005,6 +1045,61 @@ jobs:$SHARED_WHEELS \$SKIP+ # BEFORE THE RECHECK, finish what an earlier run started.+ #+ # Before and not after: this can take fifteen minutes of downloads, and a+ # recheck on the far side of that gap is the stale answer it exists to+ # replace, since the Studio workflow can take the account's last session+ # meanwhile. Collecting first also frees whatever slot a finished kernel+ # held.+ #+ # The cheap half of collection: this job already has a runner and this+ # account's token, so a previous commit's finished kernel costs seconds+ # and publishes against ITS OWN sha. `kaggle-collect.yml` does the same on+ # a schedule, always but only every ten minutes.+ #+ # It also answers whether a kernel for THIS commit is already in flight,+ # so a re-run does not buy a second session for a result already coming.+ - name: Collect finished Kaggle runs+ id: collect+ if: steps.ref.outputs.stand_down != 'true' && steps.pins.outputs.stand_down != 'true' && steps.stepcount.outputs.stand_down != 'true'+ env:+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}+ run: |+ python .github/scripts/kaggle_t4_ci/collect.py \+ --outdir kaggle_collected \+ --expect '${{ steps.build.outputs.payloads }}' \+ --sha '${{ steps.ref.outputs.ref }}' \+ --kind notebook \+ --slot '${{ inputs.slot || '1' }}' \+ --no-delete \+ --target-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"++ # Post FIRST, delete SECOND: a verdict that fails to reach GitHub leaves+ # its kernel up for the next pass, and deleting first would make that+ # failure permanent. The poster is shared by all three workflows; it+ # expands the slug's abbreviated sha (the statuses API 422s one) and hands+ # every field to `gh` as its own argument, never through a shell.+ - name: Post the collected verdicts as commit statuses+ id: post+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ python .github/scripts/kaggle_t4_ci/post_statuses.py \+ --result kaggle_collected/collect_result.json \+ --out kaggle_collected/posted.json++ - name: Release the kernels whose verdicts were delivered+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''+ env:+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}+ run: |+ python .github/scripts/kaggle_t4_ci/collect.py \+ --outdir kaggle_collected \+ --delete-collected kaggle_collected/collect_result.json \+ --posted kaggle_collected/posted.json+# kernel-timeout-sec is passed to Kaggle at push time as a BACKSTOP, not# the budget control. Measured on 2026-08-11: a kernel pushed with -t 5400# whose nbconvert crashed sat in RUNNING for over two hours and stopped@@ -1015,6 +1110,11 @@ jobs:# takes it per push and the larger requirement wins: kernel 2 downloads a# 12GB gpt-oss checkpoint before it trains anything.#+ # `--sha` stands a run down when a kernel of this commit is already up,+ # and `--slot` is part of that identity: slot 2 is the opt-in for a SECOND+ # session on the same ref, so a slot-1 kernel does not count against it+ # while a retry of the slot-2 run itself does.+ ## THE GATE'S ANSWER CAN BE STALE BY THE TIME WE GET HERE, and this is the# first point at which it cannot be. The `gate` job surveys quota and# in-flight kernels before this job exists, and this job then queues on@@ -1064,6 +1164,11 @@ jobs:run: |echo "::warning title=Stood down at the second check::${REASON}"+ - name: Note that this commit is already running on Kaggle+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight == 'true'+ run: |+ echo "::notice title=Already in flight::a Kaggle kernel for this commit is still running from an earlier run, so nothing was dispatched. Its result will arrive as the kaggle-t4-notebook commit status."+# --deadline-epoch is when THIS JOB is killed, and the launcher pushes# nothing unless its whole worst case still fits before it. Everything# above is unbounded -- a checkout, a pip install off a slow index, the@@ -1080,9 +1185,19 @@ jobs:# JOB_TIMEOUT_MINUTES restates this job's timeout-minutes because a job# cannot read its own; the harness suite asserts the two agree, so moving# one without the other is a red test rather than a silent gap.- - name: Launch on Kaggle and collect+ #+ # PUSH AND EXIT. Waiting here cost 41.5 minutes of runner per commit+ # (measured, run 33479481067) against a queue 400-670 deep.+ #+ # This job now succeeds by DISPATCHING and proves only that Kaggle+ # accepted a push. The verdict arrives later as the `kaggle-t4-notebook`+ # commit status, which cannot be a REQUIRED check while the gate samples+ # and the workflow path-filters: a skipped commit never gets one.+ # --commit-sha puts the sha into the slug, the only record outliving this+ # runner.+ - name: Dispatch to Kaggleid: launch- if: steps.recheck.outputs.should_run == 'true'+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight != 'true'env:KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}JOB_TIMEOUT_MINUTES: 260@@ -1095,8 +1210,28 @@ jobs:--kernel-timeout-sec 3600 \--max-wait 5400 \--poll-every 60 \+ --dispatch \+ --kind notebook \+ --slot '${{ inputs.slot || '1' }}' \+ --commit-sha '${{ steps.ref.outputs.ref }}' \--deadline-epoch "$(( JOB_START_EPOCH + JOB_TIMEOUT_MINUTES * 60 ))"+ # Gated on the launcher's verdict, not its exit code: launch.py exits 0+ # for every stand-down, and a pending status for a kernel that does not+ # exist is one no collector can ever replace.+ - name: Mark the dispatch pending on this commit+ if: steps.launch.outputs.verdict == 'dispatched'+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ # A required context that has never been posted blocks a pull request+ # forever with no explanation on the page.+ gh api "repos/$GITHUB_REPOSITORY/statuses/${{ steps.ref.outputs.ref }}" \+ -f state=pending -f context=kaggle-t4-notebook \+ -f description="dispatched to Kaggle; the result is collected when the kernel finishes" \+ -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \+ --silent || echo "::warning title=Could not post the pending status::kaggle-t4-notebook"+# The launcher collects executed notebooks and the kernel log and nothing# else, deliberately. The Studio payload therefore encodes its# screenshots and logs into its own cell output; this puts them back.@@ -1105,12 +1240,19 @@ jobs:# It is safe on a run that carried no Studio payload: the collector finds# no encoded evidence and writes nothing.- name: Unpack the Playwright evidence- if: always() && steps.recheck.outputs.should_run == 'true'+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb', 'kaggle_collected/**/*_output.ipynb') != ''continue-on-error: truerun: |- python .github/scripts/kaggle_studio_ci/collect_evidence.py \- --evidence kaggle_evidence \- --outdir studio_evidence+ # One kernel per call: the bundle is reassembled from numbered chunks+ # and one walk over two would splice them together. Both trees, since+ # a waited-on kernel lands in kaggle_evidence and a collected one in+ # kaggle_collected.+ for dir in kaggle_evidence/*/ kaggle_collected/*/; do+ [ -d "$dir" ] || continue+ python .github/scripts/kaggle_studio_ci/collect_evidence.py \+ --evidence "$dir" \+ --outdir "studio_evidence/$(basename "$dir")"+ done# always(): the evidence is most valuable when something went wrong, and# the executed notebooks are kilobytes.@@ -1136,6 +1278,7 @@ jobs:name: kaggle-t4-evidencepath: |kaggle_evidence/**+ kaggle_collected/**studio_evidence/**kernel*.ipynb# 14 rather than the repo's usual 7: this job runs on a minority of@@ -1157,7 +1300,7 @@ jobs:# recheck has no evidence, and "0 of 4 payloads" would read as a failure# rather than the skip it is.- name: Report- if: always() && steps.recheck.outputs.should_run == 'true'+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb') != ''run: |python .github/scripts/kaggle_t4_ci/report.py \--evidence kaggle_evidence \@@ -1175,7 +1318,7 @@ jobs:# report set with `partial` rather than silence, so ungated it would print# a PARTIAL section on every leg-list run about a payload it never ran.- name: Report Studio- if: always() && steps.recheck.outputs.should_run == 'true' && steps.build.outputs.studio == 'true'+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb') != '' && steps.build.outputs.studio == 'true'run: |python .github/scripts/kaggle_studio_ci/report.py \--evidence kaggle_evidencediff --git a/.github/workflows/kaggle-t4-studio-gpu-ci.yml b/.github/workflows/kaggle-t4-studio-gpu-ci.ymlindex 64bf7301409..952b5f7a7ec 100644--- a/.github/workflows/kaggle-t4-studio-gpu-ci.yml+++ b/.github/workflows/kaggle-t4-studio-gpu-ci.yml@@ -276,7 +276,10 @@ jobs:&& (github.event.action != 'labeled'|| github.event.label.name == 'kaggle-studio-gpu-ci')runs-on: ubuntu-latest- timeout-minutes: 10+ # 15, the worst case with two accounts: each one's authentication, username+ # and quota read at the socket ceiling, one shared survey budget, and the+ # call still in flight when it expires. The suite asserts the arithmetic.+ timeout-minutes: 15outputs:should_run: ${{ steps.decide.outputs.should_run }}reason: ${{ steps.decide.outputs.reason }}@@ -286,6 +289,9 @@ jobs:account: ${{ steps.decide.outputs.account }}account_user: ${{ steps.decide.outputs.account_user }}matrix: ${{ steps.decide.outputs.matrix }}+ # The commit under test, resolved once: the GPU job tests THIS and never+ # re-resolves a moving ref.+ head_sha: ${{ steps.ref.outputs.head_sha }}steps:- name: Harden runner (audit)uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0@@ -311,14 +317,54 @@ jobs:# demanding two free concurrency slots would stand the job down on an# account that has room for it.#- # budget-hours is the worst case ONE invocation can cost: the 70-minute- # kernel ceiling set below, rounded up.+ # budget-hours is the worst case ONE invocation can cost. Nothing waits+ # on the kernel now, and one that ignores its own timeout (seen: two hours+ # past it, still RUNNING) is left to the collector's age ceiling plus its+ # schedule and job timeout, so the worst case is that whole reaper window.+ # The harness suite derives it.## reserve-hours is 10 rather than the notebook leg's 20, and that is the# single knob that implements "CI may use up to 50 of the 60 hours".# Raising it throttles BOTH legs, since they read the same account-wide# quota. Do not raise it above roughly 45 or CI never runs at all on a# week with any other usage.+ # THE COMMIT THE GPU JOB WILL TEST, which is not always this workflow's+ # own: a dispatch naming `unsloth_ref` tests that ref. Keyed on github.sha+ # instead, the in-flight check would look for the wrong kernel, and once+ # the default branch moved two runs of one requested ref could be drawn+ # onto different accounts and never see each other. A ref that does not+ # resolve leaves the key EMPTY, so the draw falls back to the run id and+ # the GPU job stands down on the same ref moments later.+ - name: Name the commit under test+ id: ref+ env:+ UNSLOTH_REF: ${{ inputs.unsloth_ref }}+ HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}+ run: |+ if [ -z "$UNSLOTH_REF" ]; then+ echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ if printf '%s' "$UNSLOTH_REF" | grep -Eq '^[0-9a-f]{40}$'; then+ RESOLVED="$UNSLOTH_REF"+ else+ RESOLVED=''+ for attempt in 1 2 3; do+ LS="$(git ls-remote https://github.com/unslothai/unsloth "$UNSLOTH_REF" || true)"+ PEELED="$(printf '%s\n' "$LS" | grep '\^{}$' | head -n1 | cut -f1)"+ if [ -n "$PEELED" ]; then+ RESOLVED="$PEELED"+ else+ RESOLVED="$(printf '%s\n' "$LS" | head -n1 | cut -f1)"+ fi+ if [ -n "$RESOLVED" ]; then break; fi+ echo "ls-remote attempt $attempt resolved nothing"+ sleep 10+ done+ fi+ echo "head_sha=$RESOLVED" >> "$GITHUB_OUTPUT"+ echo "the gate is keyed on unslothai/unsloth@${RESOLVED:-(unresolved)}"+- name: Decideid: decideenv:@@ -329,16 +375,25 @@ jobs:run: |python .github/scripts/kaggle_t4_ci/gate.py \--percent 5 \+ --head-sha '${{ steps.ref.outputs.head_sha }}' \+ --kind studio \--force '${{ inputs.force || 'false' }}' \--labels '${{ join(github.event.pull_request.labels.*.name, ',') }}' \--label-name kaggle-studio-gpu-ci \--kernels 1 \- --budget-hours 2 \+ --budget-hours 4 \--reserve-hours 10studio-gpu:name: Unsloth GPU smokeneeds: gate+ # Load-bearing: this job dispatches rather than waits, so it cannot BE the+ # check that says the Studio smoke passed. It posts that verdict as a status+ # on whichever commit the collected kernel belongs to, not necessarily this+ # run's.+ permissions:+ contents: read+ statuses: writeif: needs.gate.outputs.should_run == 'true'runs-on: ubuntu-latest# One element, carrying the chosen account into a context `secrets[...]`@@ -394,15 +449,55 @@ jobs:# head SHA, so the kernel exercises the proposed code and not the merge# base. Pinned to a SHA rather than a branch so a push landing mid-run# cannot change what was tested.+ # To a COMMIT, never a ref: the commit goes into the kernel's slug, the+ # only record a later collector has. A branch or tag cannot be encoded,+ # and launch.py refuses it.- name: Resolve the ref under testid: ref+ env:+ UNSLOTH_REF: ${{ inputs.unsloth_ref }}+ HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}+ GATE_SHA: ${{ needs.gate.outputs.head_sha }}run: |- REF='${{ inputs.unsloth_ref }}'- if [ -z "$REF" ]; then- REF='${{ github.event.pull_request.head.sha || github.sha }}'+ if [ -z "$UNSLOTH_REF" ]; then+ echo "ref=$HEAD_SHA" >> "$GITHUB_OUTPUT"+ echo "testing unslothai/unsloth@$HEAD_SHA"+ exit 0+ fi+ # RESOLVED ONCE, by the gate job, and reused rather than asked again:+ # a branch or tag can move while this job queues, and a second answer+ # could name a commit the gate never keyed the draw or the in-flight+ # check on, whose own run may already be out on the other account.+ RESOLVED="$GATE_SHA"+ # A full SHA is accepted on shape alone above and `git ls-remote`+ # matches refs, so neither says the repo HAS the commit. A mistyped or+ # force-pushed-away SHA would spend a session and never get a verdict.+ # Fetching the object is the reachability pip needs anyway; GitHub+ # refuses an unreachable one with exit 128.+ if ! printf '%s' "$RESOLVED" | grep -Eq '^[0-9a-f]{40}$'; then+ RESOLVED=''fi- echo "ref=$REF" >> "$GITHUB_OUTPUT"- echo "testing unslothai/unsloth@$REF"+ if [ -n "$RESOLVED" ]; then+ EXISTS=''+ for attempt in 1 2 3; do+ if git fetch --quiet --depth=1 https://github.com/unslothai/unsloth "$RESOLVED"; then+ EXISTS=yes+ break+ fi+ echo "fetch attempt $attempt did not find $RESOLVED"+ sleep 10+ done+ if [ -z "$EXISTS" ]; then+ echo "::error title=The requested unsloth commit does not exist::unslothai/unsloth does not serve the commit $RESOLVED, so a session would be spent on a commit no status can ever be posted to."+ exit 1+ fi+ fi+ if [ -z "$RESOLVED" ]; then+ echo "::error title=Could not resolve the requested unsloth ref::'$UNSLOTH_REF' names no branch, tag or full commit on unslothai/unsloth, so there is no commit to dispatch or to report against."+ exit 1+ fi+ echo "ref=$RESOLVED" >> "$GITHUB_OUTPUT"+ echo "testing unslothai/unsloth@$RESOLVED (from '$UNSLOTH_REF')"- name: Build the kernel notebookenv:@@ -428,6 +523,50 @@ jobs:--payload-args "$ARGS" \--per-run-timeout 3900+ # Finish what an earlier run started, BEFORE the recheck: this can spend+ # fifteen minutes downloading and the notebook workflow can take the+ # account's last session in that window; the recheck must be the last+ # thing before the push. As on the notebook leg, the token is already in+ # hand, collecting frees the slot this job wants, and it answers whether a+ # kernel for THIS commit is already in flight.+ - name: Collect finished Kaggle runs+ id: collect+ env:+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}+ run: |+ python .github/scripts/kaggle_t4_ci/collect.py \+ --outdir kaggle_collected \+ --expect 1 \+ --sha '${{ steps.ref.outputs.ref }}' \+ --kind studio \+ --no-delete \+ --target-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"++ # Post FIRST, delete SECOND: a verdict that fails to reach GitHub leaves+ # its kernel up for the next pass, and deleting first would make that+ # failure permanent. The poster is shared by all three workflows; it+ # expands the slug's abbreviated sha (the statuses API 422s one) and hands+ # every field to `gh` as its own argument, never through a shell.+ - name: Post the collected verdicts as commit statuses+ id: post+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ python .github/scripts/kaggle_t4_ci/post_statuses.py \+ --result kaggle_collected/collect_result.json \+ --out kaggle_collected/posted.json++ - name: Release the kernels whose verdicts were delivered+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''+ env:+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}+ run: |+ python .github/scripts/kaggle_t4_ci/collect.py \+ --outdir kaggle_collected \+ --delete-collected kaggle_collected/collect_result.json \+ --posted kaggle_collected/posted.json+# WITH THE ACCOUNT SLOT IN HAND, and immediately before the push. The# notebook leg has always done this and this one never has; two accounts# is what makes the gap matter. A gate answer can be a whole run stale by@@ -450,7 +589,7 @@ jobs:--soft-fail \--account-env KAGGLE_API_TOKEN \--kernels 1 \- --budget-hours 2 \+ --budget-hours 4 \--reserve-hours 10# Through the environment rather than an inline expression, so nothing the@@ -462,6 +601,11 @@ jobs:run: |echo "::warning title=Stood down at the second check::${REASON}"+ - name: Note that this commit is already running on Kaggle+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight == 'true'+ run: |+ echo "::notice title=Already in flight::a Kaggle kernel for this commit is still running from an earlier run, so nothing was dispatched. Its result will arrive as the kaggle-studio-gpu commit status."+# kernel-timeout-sec is the load-bearing budget control: Kaggle itself# kills the session at that point. A runner that is cancelled cannot# clean up after itself, so the ceiling has to be enforced on Kaggle's@@ -470,12 +614,18 @@ jobs:# The launcher is shared with the notebook leg and is used unchanged:# this payload prints the same T4_SMOKE_REPORT line, so the transport# needed no modification.- - name: Launch on Kaggle and collect+ #+ # PUSH AND EXIT. Waiting here cost 18.7 minutes of runner (run+ # 33486360729). This job now succeeds by dispatching and cannot say+ # whether the Studio smoke passed; the verdict arrives as the+ # `kaggle-studio-gpu` commit status. Not a required check while the gate+ # samples: a skipped commit never gets one.+ - name: Dispatch to Kaggleid: launch# GATED ON THE RECHECK, or the recheck is decorative: a step that# measures the account and cannot stop the push is a log line, not a# control. This is the same shape the notebook leg uses.- if: steps.recheck.outputs.should_run == 'true'+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight != 'true'env:KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}run: |@@ -486,17 +636,46 @@ jobs:--expect 1 \--kernel-timeout-sec 4200 \--max-wait 5400 \- --poll-every 60+ --poll-every 60 \+ --dispatch \+ --kind studio \+ --commit-sha '${{ steps.ref.outputs.ref }}'++ # Gated on the launcher's verdict, not its exit code: launch.py exits 0+ # for every stand-down, and a pending status for a kernel that does not+ # exist is one no collector can ever replace.+ - name: Mark the dispatch pending on this commit+ if: steps.launch.outputs.verdict == 'dispatched'+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ # A pending status as soon as the kernel is queued, so a required+ # context that has not reported yet reads as waiting, not as nothing.+ gh api "repos/$GITHUB_REPOSITORY/statuses/${{ steps.ref.outputs.ref }}" \+ -f state=pending -f context=kaggle-studio-gpu \+ -f description="dispatched to Kaggle; the result is collected when the kernel finishes" \+ -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \+ --silent || echo "::warning title=Could not post the pending status::kaggle-studio-gpu"# The launcher collects executed notebooks and the kernel log and# nothing else, deliberately. The payload therefore encodes its# screenshots and logs into its own cell output; this puts them back.- name: Unpack the Playwright evidence- if: always()+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb', 'kaggle_collected/**/*_output.ipynb') != ''+ # Best effort: the verdict was posted above, and evidence Kaggle+ # mangled must not turn the job red after the fact.+ continue-on-error: truerun: |- python .github/scripts/kaggle_studio_ci/collect_evidence.py \- --evidence kaggle_evidence \- --outdir studio_evidence+ # One kernel per call: the bundle is reassembled from numbered chunks+ # and one walk over two would splice them together. Both trees, since+ # a waited-on kernel lands in kaggle_evidence and a collected one in+ # kaggle_collected.+ for dir in kaggle_evidence/*/ kaggle_collected/*/; do+ [ -d "$dir" ] || continue+ python .github/scripts/kaggle_studio_ci/collect_evidence.py \+ --evidence "$dir" \+ --outdir "studio_evidence/$(basename "$dir")"+ done# always(): the evidence is most valuable exactly when something went# wrong. Screenshots only travel on a failing run, so a passing upload@@ -508,6 +687,7 @@ jobs:name: kaggle-studio-gpu-evidencepath: |kaggle_evidence/**+ kaggle_collected/**studio_evidence/**kernel.ipynb# 14 rather than the repo's usual 7. This job runs on a minority of@@ -517,7 +697,7 @@ jobs:if-no-files-found: warn- name: Report- if: always()+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb') != ''run: |python .github/scripts/kaggle_studio_ci/report.py \--evidence kaggle_evidencediff --git a/.gitignore b/.gitignoreindex 5f406be9fae..1097bf4af7f 100644--- a/.gitignore+++ b/.gitignore@@ -254,3 +254,6 @@ Speech_12dB_s16.flac# storage_roots.studio_root at it. It is a mutable SQLite runtime database, not a fixture,# so a test run or an Unsloth start rewrites it and dirties the checkout..studio-test-root/++# launch.py --outdir scratch from a local run+/out/diff --git a/tests/kaggle/test_fire_and_forget.py b/tests/kaggle/test_fire_and_forget.pynew file mode 100644index 00000000000..35e3240c60a--- /dev/null+++ b/tests/kaggle/test_fire_and_forget.py@@ -0,0 +1,1791 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Dispatch and collect: the ways this trades forty minutes for nothing.++The GPU jobs no longer wait for their kernel, buying back 41.5 and 18.7 minutes+of runner per commit (runs 33479481067 and 33486360729). Every test here aims+at the one new class of bug, **the result stops arriving and nothing says so**,+in its four green shapes:++1. the dispatching job reports success and is still the required check, so the+ check now means "Kaggle accepted a push";+2. the kernel is never collected, so it bills quota to its ceiling and the+ commit keeps a status that never resolves;+3. the collector deletes a kernel before reading it, or one never ours;+4. the slug loses the commit, so a finished result cannot be attributed.++Everything runs on CPU against stubs; no Kaggle session is spent.+"""++from __future__ import annotations++import json+import re+import sys+import time+from pathlib import Path++import pytest+import yaml++REPO_ROOT = Path(__file__).resolve().parents[2]+CI_DIR = REPO_ROOT / ".github" / "scripts" / "kaggle_t4_ci"+WORKFLOWS = REPO_ROOT / ".github" / "workflows"+NOTEBOOK_WF = WORKFLOWS / "kaggle-t4-notebook-ci.yml"+STUDIO_WF = WORKFLOWS / "kaggle-t4-studio-gpu-ci.yml"+COLLECT_WF = WORKFLOWS / "kaggle-collect.yml"++sys.path.insert(0, str(CI_DIR))++import collect # noqa: E402+import gate # noqa: E402+import launch # noqa: E402+import post_statuses # noqa: E402+++def _wf(path: Path) -> dict:+ return yaml.safe_load(path.read_text(encoding = "utf-8"))+++def _steps(workflow: dict) -> list[tuple[str, str, dict]]:+ out = []+ for job_name, job in workflow["jobs"].items():+ for step in job.get("steps", []) or []:+ out.append((job_name, step.get("name", ""), step))+ return out+++# ------------------------------------------------------- the slug is the record+++def test_the_slug_carries_the_commit_and_the_workflow():+ """The slug is the ONLY thing surviving the dispatching runner: without the+ commit and the workflow in it, a finished kernel has nowhere to report."""+ name = launch.slug_name("notebook", "1a2b3c4d5e6f7890")+ parsed = launch.parse_slug(name)+ assert parsed is not None, f"{name} does not parse as one of ours"+ assert parsed["sha"] == "1a2b3c4d5e6f", parsed+ assert parsed["kind"] == "notebook", parsed+++def test_the_slug_carries_twelve_hex_characters_and_still_reads_eight():+ """Two commits can share eight characters, and the commits API 422s the+ ambiguous prefix on every pass, so the status stays pending forever. Twelve+ is written; eight is still read, since old kernels outlive the change."""+ assert launch.SLUG_SHA_LEN == 12+ name = launch.slug_name("studio", "a" * 40)+ assert launch.parse_slug(name)["sha"] == "a" * 12+ old = launch.parse_slug("me/unsloth-t4-ci-nabcdef01-1111")+ assert old is not None and old["sha"] == "abcdef01" and old["kind"] == "notebook"+ # Too short to be unambiguous is not written into a slug at all: it falls+ # back to the unattributable form rather than inventing a prefix.+ assert launch.parse_slug(launch.slug_name("notebook", "abcdef01"))["legacy"] is True+++def test_in_flight_matches_the_old_and_the_new_slug_forms(tmp_path, monkeypatch):+ """A kernel pushed with the eight-character form must still count as in+ flight for its full commit, and a twelve-character one for the same."""+ full = "abcdef0123456789" + "0" * 24+ for slug_sha in ("abcdef01", "abcdef012345"):+ slug = f"danielhanchen/unsloth-t4-ci-n{slug_sha}-1111"+ api = _StubApi([_StubKernel(slug)], {slug: "RUNNING"})+ monkeypatch.setattr(launch, "_api", lambda api = api: api)+ monkeypatch.setattr(+ sys,+ "argv",+ ["collect", "--outdir", str(tmp_path / slug_sha), "--sha", full, "--kind", "notebook"],+ )+ assert collect.main() == 0+ result = json.loads((tmp_path / slug_sha / "collect_result.json").read_text())+ assert result["in_flight_for_sha"] is True, (slug_sha, result)+++def test_the_slug_carries_the_slot_and_reads_slot_one_when_absent():+ """Slot 2 is a deliberate second session beside slot 1, so the duplicate+ check needs "this commit, this kind, THIS slot" and the slug has to carry+ the slot. Absent means slot 1, so every earlier slug still reads."""+ two = launch.slug_name("notebook", "1a2b3c4d5e6f7890", slot = "2")+ parsed = launch.parse_slug(two)+ assert parsed and parsed["slot"] == "2" and parsed["sha"] == "1a2b3c4d5e6f", (two, parsed)+ one = launch.slug_name("notebook", "1a2b3c4d5e6f7890", slot = "1")+ assert launch.parse_slug(one)["slot"] == "1"+ assert launch.parse_slug("me/unsloth-t4-ci-n1a2b3c4d5e6f-abcd")["slot"] == "1"+ assert launch.parse_slug("me/unsloth-t4-ci-nabcdef01-1111")["slot"] == "1"+ assert launch.parse_slug("me/unsloth-t4-ci-nabcdef01-21111")["slot"] == "2"+ # Anything that is not a single digit 2-9 is slot 1 in the slug.+ assert launch.parse_slug(launch.slug_name("notebook", "1a2b3c4d5e6f", slot = "23"))["slot"] == "1"+ assert launch._slugify(two.replace("-", " ")) == two+ # The collector narrows its in-flight answer by slot too.+ assert "slot" in launch.parse_slug(two)+ assert (+ gate.in_flight_for_commit([f"me/{two} (RUNNING)"], "1a2b3c4d5e6f7890", "notebook", "2")+ == f"me/{two}"+ )+ assert (+ gate.in_flight_for_commit([f"me/{two} (RUNNING)"], "1a2b3c4d5e6f7890", "notebook", "1")+ is None+ )+ assert (+ gate.in_flight_for_commit([f"me/{one} (RUNNING)"], "1a2b3c4d5e6f7890", "notebook", "2")+ is None+ )+++def test_the_collector_in_flight_answer_is_per_slot(tmp_path, monkeypatch):+ full = "abcdef0123456789" + "0" * 24+ slug = "danielhanchen/unsloth-t4-ci-nabcdef012345-21111"+ for slot, expected in (("2", True), ("1", False)):+ api = _StubApi([_StubKernel(slug)], {slug: "RUNNING"})+ monkeypatch.setattr(launch, "_api", lambda api = api: api)+ out = tmp_path / f"slot{slot}"+ monkeypatch.setattr(+ sys,+ "argv",+ ["collect", "--outdir", str(out), "--sha", full, "--kind", "notebook", "--slot", slot],+ )+ assert collect.main() == 0+ result = json.loads((out / "collect_result.json").read_text())+ assert result["in_flight_for_sha"] is expected, (slot, result)+++def test_two_dispatches_of_one_commit_do_not_collide():+ """A re-run, or slots 1 and 2, must not push to one slug: pushing to an+ existing id starts a second session the status calls never answer for."""+ names = {launch.slug_name("notebook", "deadbeefcafe") for _ in range(200)}+ assert len(names) > 190, f"only {len(names)} distinct slugs in 200 draws"+++def test_the_slug_still_round_trips_through_slugify():+ """Kaggle files the kernel under the slugified TITLE, not the metadata id,+ so a name that does not round-trip 403s every later status call."""+ for kind in ("notebook", "studio"):+ name = launch.slug_name(kind, "0123456789ab")+ assert launch._slugify(name.replace("-", " ")) == name, name+++def test_the_gate_still_recognises_a_dispatched_kernel_as_ours():+ """THE ONE THAT KEEPS THE CONCURRENCY CONTROL WORKING. The GitHub group no+ longer bounds Kaggle sessions; the gate's survey of slugs carrying+ OWN_KERNEL_PREFIX does, and without the prefix every push believes the+ account idle until Kaggle refuses one at its 2-session cap."""+ import gate+ for kind in ("notebook", "studio"):+ name = launch.slug_name(kind, "abcdef01")+ assert name.startswith(gate.OWN_KERNEL_PREFIX), (+ f"{name} does not start with the gate's {gate.OWN_KERNEL_PREFIX!r}, so the "+ "gate would count a dispatched kernel as somebody else's or not at all"+ )+++ "foreign",+ (+ "danielhanchen/unsloth-probe-vision-train-r3-6dd742",+ "someone/my-important-notebook",+ "danielhanchen/unsloth-t4-cixyz",+ "danielhanchen/unsloth-studio-ci-12345678",+ "",+ ),+)+def test_a_kernel_that_is_not_ours_is_invisible(foreign):+ """The collector enumerates a whole ACCOUNT shared with a human and DELETES+ what it collects, so there is deliberately no heuristic: a false positive+ deletes somebody's work and looks exactly like a kernel that finished."""+ assert launch.parse_slug(foreign) is None, f"{foreign!r} was accepted as one of ours"+++def test_a_legacy_slug_is_still_reapable_but_reports_nothing():+ """Kernels pushed before this change outlive it: still ours and reapable, or+ they bill forever, but with no commit to attribute a verdict to."""+ parsed = launch.parse_slug("danielhanchen/unsloth-t4-ci-d7faf2b8")+ assert parsed is not None and parsed["legacy"] is True+ assert parsed["sha"] is None and parsed["kind"] is None+ assert collect.statuses_from([{**parsed, "slug": "x", "verdict": "pass", "reason": "ok"}]) == []+++# ------------------------------------------------------------ dispatch refuses+++def test_dispatch_without_a_commit_is_a_usage_error(monkeypatch, capsys):+ """A dispatch whose slug carries no commit runs, costs quota, and reports to+ nobody. Refused at the only moment it is still free."""+ monkeypatch.setattr(+ sys,+ "argv",+ ["launch.py", "--notebook", "k.ipynb", "--outdir", "out", "--dispatch"],+ )+ with pytest.raises(SystemExit) as excinfo:+ launch.main()+ assert excinfo.value.code != 0+ assert "commit-sha" in capsys.readouterr().err+++def test_dispatch_does_not_delete_the_kernel_it_pushed():+ """The whole point: the kernel is LEFT RUNNING, through the existing+ --keep-kernel flag rather than a second condition in release(). Two guards+ on one deletion is how one ends up wrong, and this one bills GPU quota."""+ src = (CI_DIR / "launch.py").read_text(encoding = "utf-8")+ assert re.search(r"if args\.dispatch:\s*\n\s*args\.keep_kernel = True", src), (+ "dispatch mode must set keep_kernel, or release() deletes the kernel it "+ "was told to leave running and the collector finds nothing"+ )+++def test_dispatch_does_not_report_pass():+ """`dispatched` is not `pass`, and the difference is the whole change.++ Nothing has run when the dispatch returns, so `pass` would be worse than the+ silent skip this repo has been caught by twice: it looks like a result.+ """+ src = (CI_DIR / "launch.py").read_text(encoding = "utf-8")+ block = src[src.index("if args.dispatch:", src.index('result["slug"] = live[0]')) :]+ block = block[: block.index("return finish()")]+ assert '"dispatched"' in block, block[:400]+ assert 'result["verdict"] = "pass"' not in block+++def test_the_dispatch_worst_case_excludes_the_phases_it_never_runs():+ """Otherwise the pre-push guard demands two hours of job deadline for a job+ that exits in five minutes, and stands runs down for a window they do not+ need -- the opposite of what dispatch mode is for."""+ full = launch.worst_case_seconds(5400, 1)+ quick = launch.worst_case_seconds(5400, 1, dispatch = True)+ assert quick < full, (quick, full)+++# ---------------------------------------------------------------- the collector+++class _StubKernel:+ def __init__(+ self,+ ref,+ last_run_time = None,+ ):+ self.ref = ref+ self.last_run_time = last_run_time+++class _StubApi:+ CONFIG_NAME_USER = "username"++ def __init__(self, kernels, statuses):+ self.config_values = {"username": "danielhanchen"}+ self._kernels = kernels+ self._statuses = statuses+ self.status_calls: list[str] = []++ def kernels_list(+ self,+ mine = True,+ page = 1,+ page_size = 100,+ sort_by = "dateRun",+ ):+ return self._kernels if page == 1 else []++ def kernels_status(self, ref):+ self.status_calls.append(ref)+ state = self._statuses[ref]+ if isinstance(state, Exception):+ raise state+ return type("S", (), {"status": f"KernelWorkerStatus.{state}"})()+++def test_a_running_kernel_within_its_ceiling_is_left_completely_alone(tmp_path):+ """No delete, no status, not even a warning: reporting a kernel doing its+ job puts a red on a commit whose result is still coming."""+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "RUNNING"})+ deleted: list[str] = []+ entry = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 0.5,+ }+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "pending"+ assert record["deleted"] is False+ assert collect.statuses_from([record]) == [], "a running kernel must post no status"+ assert deleted == []+++def test_a_kernel_past_its_ceiling_is_reaped_and_reported(tmp_path, monkeypatch):+ """THE REASON THE SCHEDULED COLLECTOR EXISTS. Nothing deletes the kernel+ now, and a wedged one bills to its ceiling unwatched: one here was measured+ ignoring Kaggle's own `-t` timeout for over two hours. Reported as a failure+ rather than dropped, or the commit stays pending forever."""+ deleted: list[str] = []+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "RUNNING"})+ entry = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 9.0,+ }+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "reaped"+ assert deleted == ["me/unsloth-t4-ci-nabcdef01-1111"]+ status = collect.statuses_from([record])[0]+ assert status["state"] == "failure", status+++def test_a_kernel_with_no_timestamp_is_never_reaped(tmp_path, monkeypatch):+ """A missing timestamp is not evidence that a kernel is old, and guessing in+ that direction DELETES A RUNNING SESSION."""+ deleted: list[str] = []+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "RUNNING"})+ entry = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": None,+ }+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "pending"+ assert deleted == []+++def test_evidence_is_downloaded_before_the_kernel_is_deleted(tmp_path, monkeypatch):+ """A delete that lands first turns a finished run into a result nobody can+ read. Asserted by ORDER of the real calls, not by reading the source."""+ order: list[str] = []+ monkeypatch.setattr(+ launch, "fetch_evidence", lambda slug, dest, deadline = None: order.append("fetch") or {}+ )+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: order.append("delete") or True+ )+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})+ entry = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 0.4,+ }+ collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)+ assert order == ["fetch", "delete"], order+++def test_a_kernel_whose_evidence_will_not_download_is_NOT_deleted(tmp_path, monkeypatch):+ """Deleting here destroys the only copy of a finished run's result to+ reclaim a slot the kernel is not using. The next pass retries."""++ def _boom(+ slug,+ dest,+ deadline = None,+ ):+ raise TimeoutError("slow")++ deleted: list[str] = []+ monkeypatch.setattr(launch, "fetch_evidence", _boom)+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})+ entry = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 0.4,+ }+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)+ # `pending`, not `infra`: an infra verdict posts green and is released by+ # --delete-collected, breaking the "next pass retries" promise.+ assert record["verdict"] == "pending"+ assert record["verdict"] not in collect.DELETABLE+ assert collect.statuses_from([record]) == []+ assert deleted == [], "evidence that would not download must not be deleted"+++def test_an_unreadable_status_does_nothing_at_all(tmp_path, monkeypatch):+ """Both available actions are destructive: delete and we may kill a running+ session, report and we may fail a run that was fine. Ask again next pass."""+ deleted: list[str] = []+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": RuntimeError("503 upstream")})+ entry = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 0.4,+ }+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "pending"+ assert deleted == []+ assert collect.statuses_from([record]) == []+++def test_find_ours_skips_everything_it_does_not_recognise():+ """Driven against a listing mixing a human's kernels with ours: a rule fed a+ hand-written list of slugs proves nothing about the filter."""+ listing = [+ _StubKernel("danielhanchen/unsloth-probe-vision-train-r3-6dd742"),+ _StubKernel("danielhanchen/unsloth-t4-ci-nabcdef01-1111"),+ _StubKernel("danielhanchen/my-actual-work"),+ _StubKernel("danielhanchen/unsloth-t4-ci-d7faf2b8"),+ ]+ api = _StubApi(listing, {})+ ours = [k["slug"] for k in collect.find_ours(api)]+ assert ours == [+ "danielhanchen/unsloth-t4-ci-nabcdef01-1111",+ "danielhanchen/unsloth-t4-ci-d7faf2b8",+ ], ours+++def test_infra_and_partial_do_not_go_red():+ """`infra` and `partial` learned nothing about the code, and a red the+ author cannot act on is how a required check gets REMOVED."""+ assert collect.VERDICT_STATE["infra"] == "success"+ assert collect.VERDICT_STATE["partial"] == "success"+ assert collect.VERDICT_STATE["fail"] == "failure"+ assert collect.VERDICT_STATE["reaped"] == "failure"+++def test_a_failed_payload_reaches_github_as_a_failure():+ """Driven through the real verdict and status functions, since every rule+ above this one is fed a dict written by hand."""+ record = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "verdict": None,+ "reason": "",+ }+ verdict, reason = collect.verdict_of([{"passed": False, "payload": "gptoss"}], 1)+ record["verdict"], record["reason"] = verdict, reason+ status = collect.statuses_from([record])[0]+ assert status["state"] == "failure"+ assert status["context"] == "kaggle-t4-notebook"+ assert "gptoss" in status["description"]+++def test_the_collector_never_sees_a_github_token():+ """A script holding both a Kaggle and a GitHub credential is one bug away+ from sending one to the other. It emits statuses as DATA; the workflow+ posts them."""+ src = (CI_DIR / "collect.py").read_text(encoding = "utf-8")+ for forbidden in ("GH_TOKEN", "GITHUB_TOKEN", "api.github.com"):+ assert forbidden not in src, f"collect.py references {forbidden}"+++# ------------------------------------------------------------- the workflows+++ "path,kind,context",+ ((NOTEBOOK_WF, "notebook", "kaggle-t4-notebook"), (STUDIO_WF, "studio", "kaggle-studio-gpu")),+ ids = ("notebook", "studio"),+)+def test_the_gpu_job_dispatches_rather_than_waits(path, kind, context):+ """The measured win, asserted from the workflow rather than from a comment."""+ body = path.read_text(encoding = "utf-8")+ launch_calls = re.findall(r"kaggle_t4_ci/launch\.py \\\n(?:.*\n)*?(?=\n)", body)+ assert launch_calls, f"{path.name} does not invoke launch.py"+ for call in launch_calls:+ assert "--dispatch" in call, f"{path.name} still waits for its kernel:\n{call}"+ assert "--commit-sha" in call, f"{path.name} dispatches without a commit:\n{call}"+ assert f"--kind {kind}" in call, f"{path.name} dispatches without its kind:\n{call}"+++ "path,context",+ ((NOTEBOOK_WF, "kaggle-t4-notebook"), (STUDIO_WF, "kaggle-studio-gpu")),+ ids = ("notebook", "studio"),+)+def test_the_dispatching_job_can_post_the_status_that_replaces_it(path, context):+ """The job succeeds by dispatching, so the verdict has to travel some other+ way. Without `statuses: write` the collection runs and reaches nobody."""+ wf = _wf(path)+ gpu_jobs = [j for name, j in wf["jobs"].items() if name != "gate"]+ assert gpu_jobs, path.name+ for job in gpu_jobs:+ perms = job.get("permissions") or {}+ assert perms.get("statuses") == "write", (+ f"{path.name}'s GPU job cannot post commit statuses, so the verdict "+ "produced by its collection step is discarded"+ )++[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))+def test_the_gpu_job_collects_before_it_dispatches(path):+ """Order matters twice: collecting frees the session slot this job wants,+ and it answers whether this commit is already running."""+ body = path.read_text(encoding = "utf-8")+ assert "kaggle_t4_ci/collect.py" in body, f"{path.name} never collects"+ assert body.index("kaggle_t4_ci/collect.py") < body.index(+ "--dispatch"+ ), f"{path.name} dispatches before it collects"++[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))+def test_a_commit_already_in_flight_is_not_dispatched_again(path):+ """Two sessions for one result is quota spent twice, and under Kaggle's+ 2-session cap the second may take the slot the first needs."""+ for _job, name, step in _steps(_wf(path)):+ if "--dispatch" in (step.get("run") or ""):+ assert "in_flight != 'true'" in (step.get("if") or ""), (+ f"{path.name}'s dispatch step is not gated on the in-flight check: "+ f"{step.get('if')!r}"+ )+ return+ pytest.fail(f"{path.name} has no dispatch step")+++def test_a_scheduled_collector_exists_and_reaps():+ """The other route only fires when somebody pushes, so a quiet weekend with+ a wedged kernel is a weekend of unwatched billing."""+ wf = _wf(COLLECT_WF)+ on = wf.get(True) or wf.get("on")+ assert "schedule" in on, "the collector does not run on a schedule, so a quiet repo never reaps"+ assert on["schedule"], on+++def test_the_collector_covers_both_accounts():+ """The gate's weighted draw decides where a session is SPENT; it must not+ decide where results are read from."""+ wf = _wf(COLLECT_WF)+ job = wf["jobs"]["collect"]+ names = {e["secret_name"] for e in job["strategy"]["matrix"]["include"]}+ import gate++ assert names == set(gate.DEFAULT_ACCOUNT_ENVS), names+++def test_the_collector_holds_one_token_per_job():+ """As in the GPU workflows: a step holding both could authenticate as+ either, and delete kernels for an account it is not reporting for."""+ for _job, name, step in _steps(_wf(COLLECT_WF)):+ env = step.get("env") or {}+ tokens = sorted(k for k in env if k.startswith("KAGGLE_API_TOKEN"))+ if tokens:+ assert tokens == ["KAGGLE_API_TOKEN"], f"{name} sees {tokens}"+++def test_the_collector_serialises_per_account():+ """Two collectors on one account race: the loser deletes what the winner+ already deleted and reports the 404 as a failure."""+ wf = _wf(COLLECT_WF)+ conc = wf.get("concurrency")+ assert conc, "the collector has no concurrency group"+ assert conc.get("cancel-in-progress") is False, (+ "a pass midway through downloading evidence must finish; cancelling it "+ "throws away work the next pass has to redo"+ )+++def test_the_status_contexts_are_stable_strings():+ """Branch protection is configured against these names, and renaming one+ silently stops requiring it: a context that never reports is not failing."""+ assert collect.STATUS_CONTEXTS == {+ "notebook": "kaggle-t4-notebook",+ "studio": "kaggle-studio-gpu",+ }+ for path, context in ((NOTEBOOK_WF, "kaggle-t4-notebook"), (STUDIO_WF, "kaggle-studio-gpu")):+ assert context in path.read_text(encoding = "utf-8"), (+ f"{path.name} never mentions {context}, so its pending status is posted "+ "under a name nothing else uses"+ )+++def test_a_dispatch_posts_a_pending_status():+ """Without it a required context that has never reported blocks the pull+ request with no explanation, indistinguishable from an unconfigured check."""+ for path, context in ((NOTEBOOK_WF, "kaggle-t4-notebook"), (STUDIO_WF, "kaggle-studio-gpu")):+ body = path.read_text(encoding = "utf-8")+ assert re.search(+ r"state=pending -f context=" + re.escape(context), body+ ), f"{path.name} never posts a pending {context} status"+++def test_the_reporters_wait_for_an_EXECUTED_NOTEBOOK_not_just_a_directory():+ """Measured on run 33628507954, which reported `Kaggle T4 smoke: PARTIAL`+ for a dispatch where nothing had run: `hashFiles('kaggle_evidence/**')` is+ true once launch.py writes launch_result.json. The condition has to name+ what a report is MADE of, an executed notebook."""+ for path in (NOTEBOOK_WF, STUDIO_WF):+ for _job, name, step in _steps(_wf(path)):+ body = step.get("run") or ""+ if "report.py" not in body:+ continue+ condition = step.get("if") or ""+ assert "_output.ipynb" in condition, (+ f"{path.name}: step {name!r} reports on the presence of the evidence "+ f"DIRECTORY, which a dispatching run creates and does not fill: "+ f"if={condition!r}"+ )+++def _fake_gh(+ monkeypatch,+ resolve_to = None,+ post_ok = True,+ lookup_error = "",+):+ """Stand in for `gh`, recording every call. `resolve_to` is what+ `repos/../commits/<sha>` answers, None means gone in GitHub's own words, and+ `lookup_error` is what a failed lookup says."""+ calls: list[list[str]] = []++ def _gh(args):+ calls.append(list(args))+ if "/commits/" in args[1]:+ if lookup_error:+ return (1, "", lookup_error)+ if resolve_to:+ return (0, resolve_to, "")+ return (1, "", "gh: No commit found for SHA: 2ecb19df (HTTP 422)")+ return (0 if post_ok else 1, "", "")++ monkeypatch.setattr(post_statuses, "_gh", _gh)+ return calls+++_STATUS = {+ "sha": "2ecb19df",+ "state": "success",+ "context": "kaggle-t4-notebook",+ "description": "pass: all 5 payload(s) passed",+ "target_url": "https://example/run/1",+ "slug": "me/unsloth-t4-ci-n2ecb19df-1111",+ "slugs": ["me/unsloth-t4-ci-n2ecb19df-1111"],+}+++def test_an_abbreviated_sha_is_EXPANDED_before_a_status_is_posted(monkeypatch):+ """MEASURED AGAINST THE REAL API, and it fails closed in the worst way:++ POST /repos/{o}/{r}/statuses/2ecb19df+ 422 "Sha must be a valid hex object ID"++ The slug carries an abbreviation, so the poster must expand it first or+ every verdict 422s while the collection quietly succeeds.+ """+ full = "2ecb19df" + "a" * 32+ calls = _fake_gh(monkeypatch, resolve_to = full)+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")+ assert outcome["ok"] == [_STATUS["slug"]], outcome+ posts = [c for c in calls if "/statuses/" in c[1]]+ assert posts == [posts[0]] and posts[0][1] == f"repos/unslothai/unsloth/statuses/{full}"+ assert not any(c[1].endswith("/statuses/2ecb19df") for c in calls)+ # Every field is its own argument: nothing is assembled into a shell line.+ assert "-f" in posts[0] and f"description={_STATUS['description']}" in posts[0]+++def test_a_commit_that_no_longer_exists_is_reported_not_silently_dropped(monkeypatch, capsys):+ """A force-push can remove the commit a running kernel was dispatched for.+ Posting is then impossible, but it must SAY so and release the kernel rather+ than retry forever."""+ calls = _fake_gh(monkeypatch, resolve_to = None)+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")+ assert outcome["unresolved"] == [_STATUS["slug"]] and outcome["failed"] == []+ assert not any("/statuses/" in c[1] for c in calls), "posted to a commit that is gone"+ assert "Could not resolve a collected commit" in capsys.readouterr().out+++def test_every_workflow_posts_through_the_shared_poster_and_none_keeps_a_shell_loop():+ for path in (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF):+ body = path.read_text(encoding = "utf-8")+ assert "kaggle_t4_ci/post_statuses.py" in body, f"{path.name} does not use the poster"+ assert "statuses.txt" not in body, f"{path.name} still carries the tab-delimited loop"+++# ---------------------------------------------- post, then delete; never the reverse+++ "path", (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF), ids = ("notebook", "studio", "collect")+)+def test_collection_never_deletes_and_the_release_step_comes_after_posting(path):+ """A verdict that does not reach GitHub must leave its kernel up for the+ next pass: collect --no-delete, post, then --delete-collected releases only+ what was delivered."""+ body = path.read_text(encoding = "utf-8")+ steps = [(name, step) for _job, name, step in _steps(_wf(path))]+ collect_steps = [+ s+ for _n, s in steps+ if "kaggle_t4_ci/collect.py" in (s.get("run") or "")+ and "--delete-collected" not in s["run"]+ ]+ assert collect_steps, f"{path.name} never collects"+ for step in collect_steps:+ assert (+ "--no-delete" in step["run"]+ ), f"{path.name} collects with deletion on:\n{step['run']}"+ names = [n for n, _s in steps]+ post = next(i for i, (n, s) in enumerate(steps) if "post_statuses.py" in (s.get("run") or ""))+ release = next(+ i for i, (n, s) in enumerate(steps) if "--delete-collected" in (s.get("run") or "")+ )+ assert (+ release > post+ ), f"{path.name} releases kernels before posting: {names[release]!r} before {names[post]!r}"+ assert "--posted kaggle_collected/posted.json" in steps[release][1]["run"]+++def test_a_kernel_whose_status_did_not_post_is_KEPT_for_the_next_pass(tmp_path, monkeypatch):+ deleted: list[str] = []+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ result = tmp_path / "collect_result.json"+ result.write_text(+ json.dumps(+ {+ "kernels": [+ {"slug": "me/ok", "verdict": "pass", "deleted": False},+ {"slug": "me/refused", "verdict": "fail", "deleted": False},+ {"slug": "me/legacy", "verdict": "reaped", "deleted": False},+ {"slug": "me/running", "verdict": "pending", "deleted": False},+ ],+ "statuses": [],+ }+ )+ )+ posted = tmp_path / "posted.json"+ posted.write_text(json.dumps({"ok": ["me/ok"], "failed": ["me/refused"], "unresolved": []}))+ assert collect.delete_collected(result, posted) == 0+ assert deleted == ["me/ok", "me/legacy"], deleted+ outcome = json.loads((tmp_path / "delete_result.json").read_text())+ assert outcome["kept"] == ["me/refused"]+++def test_no_delivery_record_at_all_keeps_every_kernel_that_had_something_to_post(+ tmp_path, monkeypatch+):+ """The poster never ran, so deleting would lose every verdict of the pass."""+ deleted: list[str] = []+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ result = tmp_path / "collect_result.json"+ result.write_text(+ json.dumps(+ {+ "kernels": [+ {"slug": "me/a", "verdict": "pass", "deleted": False},+ {"slug": "me/nothing-to-post", "verdict": "infra", "deleted": False},+ ],+ "statuses": [{"slug": "me/a", "slugs": ["me/a"]}],+ }+ )+ )+ collect.delete_collected(result, tmp_path / "posted.json")+ assert deleted == ["me/nothing-to-post"], deleted+++def test_a_refused_post_is_red_and_recorded(monkeypatch, tmp_path):+ _fake_gh(monkeypatch, resolve_to = "2ecb19df" + "b" * 32, post_ok = False)+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")+ assert outcome["failed"] == [_STATUS["slug"]]+ result = tmp_path / "r.json"+ result.write_text(json.dumps({"statuses": [dict(_STATUS)]}))+ monkeypatch.setattr(+ sys,+ "argv",+ [+ "post_statuses",+ "--result",+ str(result),+ "--out",+ str(tmp_path / "p.json"),+ "--repo",+ "o/r",+ ],+ )+ assert post_statuses.main() == 1+ assert json.loads((tmp_path / "p.json").read_text())["failed"] == [_STATUS["slug"]]+++def test_the_poster_signs_only_records_it_recognises(monkeypatch, capsys):+ calls = _fake_gh(monkeypatch, resolve_to = "2ecb19df" + "c" * 32)+ bad_state = dict(_STATUS, state = "green")+ bad_context = dict(_STATUS, context = "ci/somebody-else")+ outcome = post_statuses.post_all([bad_state, bad_context], "o/r")+ assert outcome["invalid"] == [_STATUS["slug"], _STATUS["slug"]]+ assert calls == [], "a malformed record reached gh"+++# ------------------------------------------------- what a status is built from+++def test_two_kernels_for_one_commit_and_context_post_ONE_status_and_a_failure_wins():+ """Slots 1 and 2 on one sha are two kernels under one context, and two+ statuses would race, the last posted becoming the visible verdict."""+ records = [+ {+ "slug": "me/one",+ "sha": "abcdef01",+ "kind": "notebook",+ "verdict": "pass",+ "reason": "all passed",+ },+ {+ "slug": "me/two",+ "sha": "abcdef01",+ "kind": "notebook",+ "verdict": "fail",+ "reason": "1 of 5 failed",+ },+ {+ "slug": "me/other",+ "sha": "abcdef02",+ "kind": "notebook",+ "verdict": "pass",+ "reason": "all passed",+ },+ ]+ statuses = collect.statuses_from(records)+ assert len(statuses) == 2+ joint = next(s for s in statuses if s["sha"] == "abcdef01")+ assert joint["state"] == "failure" and sorted(joint["slugs"]) == ["me/one", "me/two"]+ # Order must not matter: the failure wins whichever kernel was listed first.+ statuses = collect.statuses_from(list(reversed(records)))+ assert next(s for s in statuses if s["sha"] == "abcdef01")["state"] == "failure"+++def test_a_description_is_one_line():+ record = {+ "slug": "me/k",+ "sha": "abcdef01",+ "kind": "notebook",+ "verdict": "fail",+ "reason": "a\nb\tc",+ }+ assert collect.statuses_from([record])[0]["description"] == "fail: a b c"+++# ------------------------------------ the evidence decides, and only complete evidence+++def _terminal_entry():+ return {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 0.4,+ }+++def test_an_incomplete_download_judges_nothing_and_keeps_the_kernel(tmp_path, monkeypatch):+ """fetch_evidence flags a spent budget instead of raising. Judging the short+ set reads a run that lost half its notebooks as whatever survived, and+ deleting makes that permanent."""+ deleted: list[str] = []+ monkeypatch.setattr(+ launch,+ "fetch_evidence",+ lambda slug, dest, deadline = None: {"notebooks": ["a"], "truncated": True},+ )+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "pending", record+ assert deleted == []+ assert collect.statuses_from([record]) == []+++def test_a_kernel_another_collector_finished_first_posts_nothing(tmp_path, monkeypatch):+ """Three unserialised workflows collect on one account, and the loser of+ that race must not post an `infra` over the winner's real verdict."""++ def _gone(+ slug,+ dest,+ deadline = None,+ ):+ raise RuntimeError(+ "404 Client Error: Not Found for url: https://www.kaggle.com/api/v1/kernels/output"+ )++ deleted: list[str] = []+ monkeypatch.setattr(launch, "fetch_evidence", _gone)+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "gone", record+ assert deleted == [] and collect.statuses_from([record]) == []+++def test_an_unreadable_report_is_infra_not_a_crash(tmp_path, monkeypatch):+ """Raising here writes no result and wedges every later pass. Nothing was+ learned, so `infra`, and the kernel is released."""++ def _boom(dest):+ raise AttributeError("'list' object has no attribute 'get'")++ monkeypatch.setattr(+ launch,+ "fetch_evidence",+ lambda slug, dest, deadline = None: {"notebooks": ["a"], "truncated": False},+ )+ monkeypatch.setattr(launch, "extract_reports", _boom)+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "infra" and "could not be read" in record["reason"]+ assert record["deleted"] is True+++def test_a_malformed_report_line_is_skipped_by_the_extractor(tmp_path):+ (tmp_path / "kernel.log").write_text(+ 'T4_SMOKE_REPORT []\nT4_SMOKE_REPORT "text"\nT4_SMOKE_REPORT {"label": "ok", "passed": true}\n',+ encoding = "utf-8",+ )+ assert launch.extract_reports(tmp_path) == [{"label": "ok", "passed": True}]+++def test_the_expected_report_count_travels_inside_the_kernel(tmp_path, monkeypatch):+ """The scheduled collector has no workflow output to read a payload count+ from, and judging a five-payload kernel against one turns a run that lost+ four into a pass. The driver's own sentinel is the record."""++ def _fetch(+ slug,+ dest,+ deadline = None,+ ):+ dest.mkdir(parents = True, exist_ok = True)+ (dest / "kernel.log").write_text(+ 'KAGGLE_T4_CI_DRIVER_EXPECT {"reports": 5}\nT4_SMOKE_REPORT {"label": "a", "passed": true}\n',+ encoding = "utf-8",+ )+ return {"notebooks": [], "log": "kernel.log", "truncated": False}++ monkeypatch.setattr(launch, "fetch_evidence", _fetch)+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["expected"] == 5+ assert record["verdict"] == "partial", record+ # And with no sentinel the caller's number stands.+ (tmp_path / "plain").mkdir()+ assert collect.expected_reports(tmp_path / "plain", 3) == 3+++def test_the_built_driver_records_its_own_expected_report_count():+ sys.path.insert(0, str(CI_DIR))+ import build_kernel++ payloads = {+ "t4_a.ipynb": {"cells": []},+ "t4_b.ipynb": {"cells": []},+ "studio_install.ipynb": {"cells": []},+ }++ def _source(nb):+ return "".join("".join(cell.get("source", [])) for cell in nb["cells"])++ driver = _source(build_kernel.build_driver(payloads, 60, cpu_lane = "studio_install.ipynb"))+ assert 'KAGGLE_T4_CI_DRIVER_EXPECT " + json.dumps({"reports": 2})' in driver+ plain = _source(build_kernel.build_driver({"t4_a.ipynb": {"cells": []}}, 60))+ assert 'json.dumps({"reports": 1})' in plain+++def test_the_scheduled_collector_no_longer_guesses_the_payload_count():+ body = COLLECT_WF.read_text(encoding = "utf-8")+ assert (+ "--expect" not in body+ ), "the scheduled collector still applies one flat --expect to every kernel"+++# ------------------------------------------------------------- the reaper's reach+++def test_a_kernel_far_past_the_ceiling_is_still_seen(monkeypatch):+ """A pass delayed six hours by an outage still finds the wedge it exists+ for; ageing it out of the listing leaves it billing with its commit+ pending forever."""+ from datetime import datetime, timedelta++ now = datetime(2026, 9, 6, 12, 0, 0)+ old = _StubKernel("me/unsloth-t4-ci-nabcdef01-1111", last_run_time = now - timedelta(hours = 30))+ api = _StubApi([old], {})+ found = collect.find_ours(api, now = now, max_age_hours = 3.0)+ assert [f["slug"] for f in found] == ["me/unsloth-t4-ci-nabcdef01-1111"]+ assert found[0]["age_hours"] == pytest.approx(30.0)+++class _PagedApi(_StubApi):+ """kernels_list answering from a list of pages."""++ def __init__(self, pages, statuses):+ super().__init__([], statuses)+ self._pages = pages+ self.pages_asked: list[int] = []++ def kernels_list(+ self,+ mine = True,+ page = 1,+ page_size = 100,+ sort_by = "dateRun",+ ):+ self.pages_asked.append(page)+ return self._pages[page - 1] if page <= len(self._pages) else []+++def test_a_kernel_of_ours_below_five_hundred_newer_records_is_still_reached():+ """The account is shared and human kernels are never deleted, so a fixed+ page cap would leave a kernel of ours below it unreachable forever. The walk+ stops only at a page entirely past the horizon with none of ours on it."""+ from datetime import datetime, timedelta++ now = datetime(2026, 9, 6, 12, 0, 0)+ recent = now - timedelta(hours = 1)+ human = [+ [_StubKernel(f"me/human-notebook-{p}-{i}", last_run_time = recent) for i in range(100)]+ for p in range(6)+ ]+ ours = _StubKernel(+ "me/unsloth-t4-ci-nabcdef012345-1111", last_run_time = now - timedelta(hours = 5)+ )+ pages = human + [[ours] + [_StubKernel(f"me/old-{i}", last_run_time = recent) for i in range(99)]]+ api = _PagedApi(pages, {})+ found = collect.find_ours(api, now = now)+ assert [f["slug"] for f in found] == [ours.ref]+ assert 7 in api.pages_asked+++def test_the_listing_walk_ends_at_a_page_past_the_horizon_with_none_of_ours():+ from datetime import datetime, timedelta++ now = datetime(2026, 9, 6, 12, 0, 0)+ ancient = now - timedelta(hours = collect.LISTING_HORIZON_HOURS + 1)+ pages = [+ [_StubKernel(f"me/human-{i}", last_run_time = now - timedelta(hours = 1)) for i in range(100)],+ [_StubKernel(f"me/ancient-{i}", last_run_time = ancient) for i in range(100)],+ [_StubKernel("me/unsloth-t4-ci-nabcdef012345-2222", last_run_time = ancient)],+ ]+ api = _PagedApi(pages, {})+ assert collect.find_ours(api, now = now) == []+ assert api.pages_asked == [1, 2], "the walk did not stop at the first page past the horizon"+ # A page past the horizon that still carries one of ours keeps the walk going.+ pages[1][50] = _StubKernel("me/unsloth-t4-ci-nabcdef012345-3333", last_run_time = ancient)+ api = _PagedApi(pages, {})+ found = collect.find_ours(api, now = now)+ assert {f["slug"] for f in found} == {+ "me/unsloth-t4-ci-nabcdef012345-3333",+ "me/unsloth-t4-ci-nabcdef012345-2222",+ }+++def test_the_listing_walk_stops_at_the_pass_deadline(monkeypatch):+ from datetime import datetime, timedelta++ now = datetime(2026, 9, 6, 12, 0, 0)+ pages = [+ [+ _StubKernel(f"me/human-{p}-{i}", last_run_time = now - timedelta(hours = 1))+ for i in range(100)+ ]+ for p in range(5)+ ]+ api = _PagedApi(pages, {})+ clock = [1000.0]+ monkeypatch.setattr(collect.time, "time", lambda: clock[0])+ orig = api.kernels_list++ def slow(*a, **k):+ clock[0] += 400.0+ return orig(*a, **k)++ api.kernels_list = slow+ collect.find_ours(api, now = now, deadline = 1500.0)+ assert api.pages_asked == [1, 2], api.pages_asked+ source = (CI_DIR / "collect.py").read_text(encoding = "utf-8")+ assert "find_ours(api, max_age_hours = args.max_age_hours, deadline = deadline)" in source+++def test_a_reaped_kernel_is_not_reported_deleted_before_it_is(tmp_path, monkeypatch):+ """The delete happens in a later step that can be refused, and this reason+ is posted as a durable status before it runs, so it must not claim it."""+ from datetime import datetime, timedelta++ slug = "me/unsloth-t4-ci-nabcdef012345-1111"+ api = _StubApi([], {slug: "RUNNING"})+ entry = {+ "slug": slug,+ "sha": "abcdef012345",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 5.0,+ }+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0, delete = False)+ assert record["verdict"] == "reaped"+ assert "released for deletion" in record["reason"] and "was deleted" not in record["reason"]+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0, delete = True)+ assert "was deleted" in record["reason"]+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: False)+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0, delete = True)+ assert "was deleted" not in record["reason"] and record["deleted"] is False+++def test_a_malformed_neighbour_notebook_does_not_hide_a_failing_report(tmp_path, monkeypatch):+ """One malformed notebook among several used to make the reader raise, and+ the collector then judged the kernel infra, posted success and released it.+ The failing report in the well-formed neighbour must survive."""+ slug = _ENTRY["slug"]+ dest = tmp_path / slug.rsplit("/", 1)[-1]+ dest.mkdir()+ failing = launch.RESULT_PREFIX + json.dumps({"label": "control", "passed": False, "model": "m"})+ (dest / f"a{launch.OUTPUT_SUFFIX}").write_text(+ json.dumps({"cells": [{"outputs": [{"text": failing + "\n"}]}]}), encoding = "utf-8"+ )+ (dest / f"b{launch.OUTPUT_SUFFIX}").write_text("[]", encoding = "utf-8")+ (dest / f"c{launch.OUTPUT_SUFFIX}").write_text(+ json.dumps(+ {+ "cells": [+ "not a cell",+ {"outputs": ["not an output", {"text": 7}, {"text": ["x", 1]}]},+ ]+ }+ ),+ encoding = "utf-8",+ )+ assert launch.extract_reports(dest) == [{"label": "control", "passed": False, "model": "m"}]+ monkeypatch.setattr(launch, "fetch_evidence", lambda slug, dest, deadline = None: {})+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)+ api = _StubApi([], {slug: "COMPLETE"})+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "fail", record+ assert "report_error" not in record+++# ------------------------------------------------------- who is allowed to be quiet+++def test_the_scheduled_collector_fails_loudly_when_it_cannot_authenticate(+ tmp_path, monkeypatch, capsys+):+ """Its token is the repository's own, so a green pass collecting nothing+ hides an expired credential while kernels bill and commits stay pending."""++ def _no(*a, **k):+ raise OSError("Could not find kaggle.json")++ monkeypatch.setattr(launch, "_api", _no)+ monkeypatch.setenv("KAGGLE_API_TOKEN", "a-token-kaggle-refused")+ monkeypatch.setattr(sys, "argv", ["collect", "--outdir", str(tmp_path), "--require-auth"])+ assert collect.main() == 1+ assert "Kaggle authentication failed" in capsys.readouterr().out+ monkeypatch.setattr(sys, "argv", ["collect", "--outdir", str(tmp_path)])+ assert collect.main() == 0, "a fork job with the secret withheld is still a skip"+++def test_an_unconfigured_collector_account_warns_and_passes(tmp_path, monkeypatch, capsys):+ """The matrix is static, so a repository with one Kaggle account leaves the+ second secret unset; that leg used to go red every ten minutes for an+ account that does not exist. An empty token is an absent account, warned+ about and green; a token present and refused is still red."""++ def _no(*a, **k):+ raise OSError("Could not find kaggle.json")++ monkeypatch.setattr(launch, "_api", _no)+ monkeypatch.delenv("KAGGLE_API_TOKEN", raising = False)+ monkeypatch.setattr(sys, "argv", ["collect", "--outdir", str(tmp_path), "--require-auth"])+ assert collect.main() == 0+ out = capsys.readouterr().out+ assert "::warning title=Kaggle account not configured" in out+ assert "::error" not in out+ monkeypatch.setenv("KAGGLE_API_TOKEN", "")+ assert collect.main() == 0+ monkeypatch.setenv("KAGGLE_API_TOKEN", "present")+ assert collect.main() == 1+ body = COLLECT_WF.read_text(encoding = "utf-8")+ assert "EMPTY token" in body+++def test_the_scheduled_workflow_asks_for_that():+ body = COLLECT_WF.read_text(encoding = "utf-8")+ assert "--require-auth" in body+++def test_the_collector_installs_the_client_the_gpu_workflows_install():+ """1.7.4.5 cannot read KAGGLE_API_TOKEN at all (#9535), so a scheduled pass+ on it authenticated nothing and, before --require-auth, said so to nobody."""+ pins = {}+ for path in (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF):+ found = re.findall(+ r"pip install [^\n]*'kaggle==([0-9][^']*)'", path.read_text(encoding = "utf-8")+ )+ assert found, f"{path.name} pins no kaggle client"+ pins[path.name] = set(found)+ assert (+ len(set().union(*pins.values())) == 1+ ), f"the workflows disagree on the kaggle client: {pins}"+++# -------------------------------------------------- what a dispatch is allowed to be+++def test_dispatch_refuses_a_ref_that_is_not_a_commit(monkeypatch, capsys):+ """`slug_name` falls back to the legacy unattributable form for anything not+ hex, so a branch here pushes a kernel that runs, bills and reports to+ nobody."""+ monkeypatch.setattr(+ sys,+ "argv",+ [+ "launch",+ "--notebook",+ "k.ipynb",+ "--outdir",+ "out",+ "--dispatch",+ "--kind",+ "studio",+ "--commit-sha",+ "main",+ ],+ )+ with pytest.raises(SystemExit) as exc:+ launch.main()+ assert exc.value.code == 2+ assert "hex commit id" in capsys.readouterr().err+++def test_the_studio_workflow_resolves_its_ref_to_a_commit_before_dispatching():+ """workflow_dispatch accepts any `unsloth_ref`; the notebook workflow+ already resolved it, the Studio one passed it straight through."""+ for path in (NOTEBOOK_WF, STUDIO_WF):+ wf = _wf(path)+ gate_ref = next(s for s in wf["jobs"]["gate"]["steps"] if s.get("id") == "ref")+ assert (+ "git ls-remote" in gate_ref["run"] and "[0-9a-f]{40}" in gate_ref["run"]+ ), f"{path.name}: the gate does not resolve refs"+ step = next(s for _j, n, s in _steps(wf) if n == "Resolve the ref under test")+ run = step["run"]+ # Resolved ONCE, by the gate, and reused: a branch re-resolved after+ # the job queued can name a commit the gate never keyed anything on.+ assert step["env"]["GATE_SHA"] == "${{ needs.gate.outputs.head_sha }}"+ assert (+ 'RESOLVED="$GATE_SHA"' in run and "$(git ls-remote" not in run+ ), f"{path.name}: the GPU job resolves the ref a second time"+ assert wf["jobs"]["gate"]["outputs"]["head_sha"] == "${{ steps.ref.outputs.head_sha }}"+ assert (+ "exit 1" in run or "stand_down=true" in run+ ), f"{path.name} dispatches on an unresolved ref"+ # A full SHA passes the shape test whether or not the repository has it;+ # only fetching the object says it is there. Without this the Studio leg+ # spent a session on a commit no status could be posted to.+ assert (+ "git fetch --quiet --depth=1 https://github.com/unslothai/unsloth" in run+ ), f"{path.name} dispatches a full SHA without checking the repository serves it"++[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))+def test_pending_is_posted_only_for_a_kernel_that_was_actually_dispatched(path):+ """launch.py exits 0 on every stand-down, and a pending status for a kernel+ that does not exist is one no collector can ever replace."""+ step = next(+ s for _j, n, s in _steps(_wf(path)) if n == "Mark the dispatch pending on this commit"+ )+ assert step.get("if") == "steps.launch.outputs.verdict == 'dispatched'", step.get("if")+++def test_a_lookup_that_could_not_be_made_keeps_the_kernel(monkeypatch, capsys):+ """Only GitHub saying the commit is not there releases a kernel: a 5xx or a+ dropped connection says nothing, and reading it as "gone" deletes the only+ copy of the result."""+ calls = _fake_gh(monkeypatch, lookup_error = "gh: HTTP 503 Service Unavailable")+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")+ assert outcome["failed"] == [_STATUS["slug"]]+ assert outcome["unresolved"] == []+ assert not any("/statuses/" in c[1] for c in calls)+ assert "Commit lookup failed" in capsys.readouterr().out+++def test_resolve_sha_separates_missing_from_unknown(monkeypatch):+ """Measured: a commit that is not there answers HTTP 422 "No commit found+ for SHA", not a 404. Anything else is an unknown, never a missing."""+ answers = {}++ def _gh(args):+ return answers[args[1]]++ monkeypatch.setattr(post_statuses, "_gh", _gh)+ answers["repos/o/r/commits/aa"] = (0, "a" * 40, "")+ answers["repos/o/r/commits/bb"] = (1, "", "gh: No commit found for SHA: bb (HTTP 422)")+ answers["repos/o/r/commits/cc"] = (1, "", "gh: HTTP 502 Bad Gateway")+ answers["repos/o/r/commits/dd"] = (1, "", "")+ # The status code alone is not the answer: 404 is also an unreadable+ # repository, 422 also an ambiguous abbreviation.+ answers["repos/o/r/commits/ee"] = (1, "", "gh: Not Found (HTTP 404)")+ answers["repos/o/r/commits/ff"] = (1, "", "gh: Validation Failed (HTTP 422)")+ assert post_statuses.resolve_sha("o/r", "aa") == ("ok", "a" * 40)+ assert post_statuses.resolve_sha("o/r", "bb") == ("missing", None)+ for sha in ("cc", "dd", "ee", "ff"):+ assert post_statuses.resolve_sha("o/r", sha) == ("error", None), sha+++# ------------------------------------------ the terminal path is for known states only+++_ENTRY = {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "legacy": False,+ "age_hours": 0.4,+}+++def test_an_unrecognised_kernel_state_is_kept_and_not_judged(tmp_path, monkeypatch):+ """Kaggle's enum can grow, and a non-terminal state has no evidence: judging+ one posts a green `infra` for a run still to come, then deletes it."""+ touched: list[str] = []+ monkeypatch.setattr(+ launch, "fetch_evidence", lambda slug, dest, deadline = None: touched.append(slug) or {}+ )+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: touched.append("delete") or True+ )+ api = _StubApi([], {_ENTRY["slug"]: "NEW_SCRIPT"})+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "pending"+ assert record["verdict"] not in collect.DELETABLE+ assert collect.statuses_from([record]) == []+ assert touched == [], "an unjudged kernel must be neither downloaded nor deleted"+++def test_a_downloaded_file_that_is_not_a_notebook_does_not_wedge_the_collector(+ tmp_path, monkeypatch+):+ """`[]` is valid JSON and not a notebook. The expected-count reader used to+ raise on it, and a collector that raises writes no result and stalls on the+ same kernel every pass."""+ slug = _ENTRY["slug"]+ dest = tmp_path / slug.rsplit("/", 1)[-1]+ dest.mkdir()+ (dest / f"x{launch.OUTPUT_SUFFIX}").write_text("[]", encoding = "utf-8")+ (dest / f"y{launch.OUTPUT_SUFFIX}").write_text(+ json.dumps({"cells": ["not a cell", {"outputs": ["not an output", {"text": 7}]}]}),+ encoding = "utf-8",+ )+ monkeypatch.setattr(launch, "fetch_evidence", lambda slug, dest, deadline = None: {})+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)+ api = _StubApi([], {slug: "COMPLETE"})+ assert collect.expected_reports(dest, 4) == 4+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 4, max_age_hours = 3.0)+ assert record["verdict"] == "infra", record+ # A notebook kernel with no expected-count record has an UNKNOWN plan.+ assert record["expected"] is None+++def test_the_evidence_download_is_clamped_to_the_pass_deadline(tmp_path, monkeypatch):+ """A kernel started just inside the pass budget must not get a fresh+ five-minute download budget, or the collector overruns the job with the+ release step still to come."""+ seen: list[float] = []++ def _fetch(+ slug,+ dest,+ deadline = None,+ ):+ seen.append(deadline)+ return {}++ monkeypatch.setattr(launch, "fetch_evidence", _fetch)+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)+ api = _StubApi([], {_ENTRY["slug"]: "COMPLETE"})+ pass_deadline = time.time() + 5.0+ collect.collect_one(+ api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0, deadline = pass_deadline+ )+ assert seen and seen[0] <= pass_deadline+ # And the collector's loop hands its deadline down at the one call site.+ source = (CI_DIR / "collect.py").read_text(encoding = "utf-8")+ body = source[source.index("deadline = time.time() + BUDGET_SEC") :]+ assert "deadline = deadline," in body[: body.index("statuses_from")]+++# ---------------------------------------- collect BEFORE the recheck, never between++[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))+def test_collection_runs_before_the_recheck_so_the_recheck_is_last_before_the_push(path):+ """Collection can spend up to BUDGET_SEC downloading, and between the+ recheck and the push that gap is the stale window the recheck exists to+ close: the other GPU workflow can take the last session in it."""+ steps = [(n, s) for _j, n, s in _steps(_wf(path))]+ names = [n for n, _s in steps]+ collect_i = names.index("Collect finished Kaggle runs")+ recheck_i = names.index("Recheck the Kaggle account")+ launch_i = names.index("Dispatch to Kaggle")+ assert collect_i < recheck_i < launch_i, names[collect_i : launch_i + 1]+ assert "steps.recheck" not in (steps[collect_i][1].get("if") or "")+ # Nothing that takes BUDGET_SEC sits between the recheck and the push.+ between = [n for n in names[recheck_i + 1 : launch_i]]+ assert not any("collect.py" in (s.get("run") or "") for n, s in steps if n in between), between+ assert "steps.recheck.outputs.should_run == 'true'" in steps[launch_i][1]["if"]+++# ------------------------------------- collected Studio evidence is unpacked too++[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))+def test_the_unpack_step_reads_the_tree_a_collected_kernel_lands_in(path):+ """A collected kernel lands in kaggle_collected/<slug>, so an unpack step+ reading only kaggle_evidence never fires on the run that retrieved the+ screenshots."""+ step = next(s for _j, n, s in _steps(_wf(path)) if n == "Unpack the Playwright evidence")+ assert "kaggle_collected/**/*_output.ipynb" in step["if"]+ assert "kaggle_evidence/**/*_output.ipynb" in step["if"]+ run = step["run"]+ assert "kaggle_collected/*/" in run and "kaggle_evidence/*/" in run+ # One kernel per call: chunks are numbered per bundle, so a walk over two+ # would splice them together.+ assert '--evidence "$dir"' in run and 'studio_evidence/$(basename "$dir")' in run+++def test_the_scheduled_collector_unpacks_and_uploads_what_it_collected():+ """It collects most Studio kernels, so it is the pass that has to put the+ screenshots back and keep them somewhere."""+ steps = [(n, s) for _j, n, s in _steps(_wf(COLLECT_WF))]+ names = [n for n, _s in steps]+ unpack = steps[names.index("Unpack the Playwright evidence")][1]+ assert "kaggle_collected/*/" in unpack["run"] and "collect_evidence.py" in unpack["run"]+ upload = steps[names.index("Upload evidence")][1]+ assert upload["uses"].startswith("actions/upload-artifact@")+ assert "kaggle_collected/**" in upload["with"]["path"]+ assert "studio_evidence/**" in upload["with"]["path"]+ assert upload.get("continue-on-error") is True, "an artifact outage is not a collection failure"+ assert names.index("Upload evidence") > names.index("Post the commit statuses")+++def test_a_listed_notebook_without_a_download_url_marks_the_evidence_incomplete(+ tmp_path, monkeypatch+):+ """Skipping it silently left `truncated` false, so the short set was judged,+ posted and its kernel deleted: a failing payload lost for good."""+ listing = {+ "files": [{"fileName": f"a{launch.OUTPUT_SUFFIX}"}],+ "log": "",+ "truncated": False,+ }+ monkeypatch.setattr(launch, "list_outputs", lambda slug, timeout = None, deadline = None: listing)+ evidence = launch.fetch_evidence("me/k", tmp_path / "k", deadline = time.time() + 60)+ assert evidence["notebooks"] == []+ assert evidence["truncated"] is True+++def test_old_and_new_slug_forms_for_one_commit_post_one_status(monkeypatch):+ """An 8 and a 12 character slug can name one commit in one pass, and only+ the commits API can say so: the collector keeps them apart and the POSTER+ merges what resolves alike, failure winning, both slugs named."""+ full = "abcdef012345" + "0" * 28+ calls: list[list[str]] = []++ def _gh(args):+ calls.append(list(args))+ if "/commits/" in args[1]:+ return (0, full, "")+ return (0, "", "")++ monkeypatch.setattr(post_statuses, "_gh", _gh)+ records = [+ {+ "slug": "me/unsloth-t4-ci-nabcdef012345-2222",+ "sha": "abcdef012345",+ "kind": "notebook",+ "verdict": "fail",+ "reason": "leg red",+ },+ {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "verdict": "pass",+ "reason": "ok",+ },+ ]+ statuses = collect.statuses_from(records)+ assert len(statuses) == 2, "the collector cannot know these are one commit"+ for order in (statuses, statuses[::-1]):+ calls.clear()+ outcome = post_statuses.post_all([dict(s) for s in order], "unslothai/unsloth")+ posts = [c for c in calls if "/statuses/" in c[1]]+ assert len(posts) == 1, posts+ assert "state=failure" in " ".join(posts[0])+ assert sorted(outcome["ok"]) == sorted(r["slug"] for r in records)+++def test_two_commits_that_merely_share_eight_characters_post_two_statuses():+ """The reason the slug grew to twelve characters: merged, one commit loses+ its verdict and has its kernel deleted as delivered."""+ records = [+ {+ "slug": "me/unsloth-t4-ci-nabcdef012345-1111",+ "sha": "abcdef012345",+ "kind": "notebook",+ "verdict": "pass",+ "reason": "ok",+ },+ {+ "slug": "me/unsloth-t4-ci-nabcdef01ffff-2222",+ "sha": "abcdef01ffff",+ "kind": "notebook",+ "verdict": "fail",+ "reason": "leg red",+ },+ ]+ statuses = collect.statuses_from(records)+ assert len(statuses) == 2, statuses+ assert {s["sha"]: s["state"] for s in statuses} == {+ "abcdef012345": "success",+ "abcdef01ffff": "failure",+ }+ assert all(len(s["slugs"]) == 1 for s in statuses)+++def test_the_release_phase_stops_at_its_budget_and_keeps_the_rest(tmp_path, monkeypatch):+ """delete_kernel allows three 180-second attempts per kernel, so an+ unbounded release could outlive the job and be killed mid-delete. What is+ left over is released by the next pass."""+ clock = [1000.0]+ monkeypatch.setattr(collect.time, "time", lambda: clock[0])++ def _slow_delete(slug, deadline = None):+ clock[0] += collect.RELEASE_BUDGET_SEC # one delete eats the whole budget+ return True++ monkeypatch.setattr(launch, "delete_kernel", _slow_delete)+ kernels = [+ {"slug": f"me/unsloth-t4-ci-nabcdef01234{i}-1111", "verdict": "pass"} for i in range(3)+ ]+ result = tmp_path / "collect_result.json"+ result.write_text(json.dumps({"kernels": kernels, "statuses": []}), encoding = "utf-8")+ posted = tmp_path / "posted.json"+ posted.write_text(json.dumps({"ok": [], "failed": []}), encoding = "utf-8")+ assert collect.delete_collected(result, posted) == 0+ outcome = json.loads((tmp_path / "delete_result.json").read_text())+ assert len(outcome["deleted"]) == 1 and len(outcome["kept"]) == 2, outcome+++def test_the_scheduled_collector_timeout_covers_collection_and_release():+ job = _wf(COLLECT_WF)["jobs"]["collect"]+ floor = (collect.BUDGET_SEC + collect.RELEASE_BUDGET_SEC) / 60 + 3+ assert job["timeout-minutes"] >= floor, (job["timeout-minutes"], floor)+++def test_nothing_tells_the_reader_to_require_a_sampled_context():+ """The gate samples and the workflows path-filter, so a commit the gate+ skips never gets a status, and a required context that never arrives blocks+ the merge."""+ for path in (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF, CI_DIR / "launch.py", CI_DIR / "collect.py"):+ assert "branch protection must require" not in path.read_text(encoding = "utf-8"), path+++def test_a_delete_is_clamped_to_the_deadline_it_is_handed(monkeypatch):+ """Checked only before the call, the deadline did not bound the release:+ two deletes of three 180-second attempts outlive a 600-second budget. Every+ attempt is clamped to what is left and none starts past it."""+ clock = [1000.0]+ timeouts: list[float] = []++ class _Proc:+ returncode = 1+ stdout = ""+ stderr = "503"++ def _run(+ cmd,+ capture_output = True,+ text = True,+ timeout = None,+ ):+ timeouts.append(timeout)+ clock[0] += timeout # the call uses its whole allowance+ return _Proc()++ monkeypatch.setattr(launch.subprocess, "run", _run)+ monkeypatch.setattr(launch.time, "time", lambda: clock[0])+ monkeypatch.setattr(launch.time, "sleep", lambda s: clock.__setitem__(0, clock[0] + s))+ assert launch.delete_kernel("me/k", deadline = 1000.0 + 200.0) is False+ assert timeouts and timeouts[0] <= 200.0+ assert sum(timeouts) <= 200.0, timeouts+ assert len(timeouts) < launch.DELETE_ATTEMPTS, "an attempt started past the deadline"+++def test_the_release_hands_every_delete_its_deadline(tmp_path, monkeypatch):+ seen: list = []+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: seen.append(deadline) or True+ )+ result = tmp_path / "collect_result.json"+ result.write_text(+ json.dumps(+ {+ "kernels": [{"slug": "me/unsloth-t4-ci-nabcdef012345-1111", "verdict": "pass"}],+ "statuses": [],+ }+ ),+ encoding = "utf-8",+ )+ posted = tmp_path / "posted.json"+ posted.write_text(json.dumps({"ok": [], "failed": []}), encoding = "utf-8")+ collect.delete_collected(result, posted)+ assert seen and seen[0] is not None and seen[0] <= time.time() + collect.RELEASE_BUDGET_SEC+++def test_the_scheduled_collector_uploads_once_and_reports_real_deletions():+ steps = [(n, s) for _j, n, s in _steps(_wf(COLLECT_WF))]+ uploads = [+ s for _n, s in steps if str(s.get("uses", "")).startswith("actions/upload-artifact@")+ ]+ assert len(uploads) == 1, "two uploads: the second one fails the job on an artifact outage"+ assert uploads[0].get("continue-on-error") is True+ assert "kaggle_collected/**" in uploads[0]["with"]["path"]+ summary = next(s for n, s in steps if n == "Summarise")+ assert (+ "delete_result.json" in summary["run"]+ ), "the summary reports the collect step's always-false deleted flag"+++def test_a_kernel_whose_status_record_was_rejected_is_kept(tmp_path, monkeypatch):+ """The poster refuses a malformed record and posts nothing for it. Keeping+ only the refused posts released that kernel with no status delivered."""+ deleted: list[str] = []+ monkeypatch.setattr(+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True+ )+ slug = "me/unsloth-t4-ci-nabcdef012345-1111"+ result = tmp_path / "collect_result.json"+ result.write_text(+ json.dumps({"kernels": [{"slug": slug, "verdict": "pass"}], "statuses": []}),+ encoding = "utf-8",+ )+ posted = tmp_path / "posted.json"+ posted.write_text(json.dumps({"ok": [], "failed": [], "invalid": [slug]}), encoding = "utf-8")+ collect.delete_collected(result, posted)+ assert deleted == []+ outcome = json.loads((tmp_path / "delete_result.json").read_text())+ assert outcome["kept"] == [slug]+++def test_the_pass_budget_starts_before_the_kernel_listing():+ """Five slow listing pages under the socket timeout are minutes. A budget+ started after them is that much later than the job timeout was sized for."""+ source = (CI_DIR / "collect.py").read_text(encoding = "utf-8")+ body = source[source.index("def main(") :]+ assert body.index("deadline = time.time() + BUDGET_SEC") < body.index("ours = find_ours(")+++def test_two_commits_that_share_eight_characters_resolve_apart_and_post_twice(monkeypatch):+ """The transition case: an old 8 character slug for commit A and a 12+ character one for a commit B starting with the same eight. Two commits."""+ a_full = "abcdef01" + "a" * 32+ b_full = "abcdef01ffff" + "b" * 28+ calls: list[list[str]] = []++ def _gh(args):+ calls.append(list(args))+ if "/commits/" in args[1]:+ sha = args[1].rsplit("/", 1)[-1]+ return (0, a_full if sha == "abcdef01" else b_full, "")+ return (0, "", "")++ monkeypatch.setattr(post_statuses, "_gh", _gh)+ statuses = collect.statuses_from(+ [+ {+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",+ "sha": "abcdef01",+ "kind": "notebook",+ "verdict": "pass",+ "reason": "ok",+ },+ {+ "slug": "me/unsloth-t4-ci-nabcdef01ffff-2222",+ "sha": "abcdef01ffff",+ "kind": "notebook",+ "verdict": "fail",+ "reason": "red",+ },+ ]+ )+ outcome = post_statuses.post_all(statuses, "unslothai/unsloth")+ posts = [c for c in calls if "/statuses/" in c[1]]+ assert len(posts) == 2+ assert {c[1].rsplit("/", 1)[-1] for c in posts} == {a_full, b_full}+ assert len(outcome["ok"]) == 2+++def test_a_notebook_kernel_without_its_expected_count_is_never_a_pass(tmp_path, monkeypatch):+ """A notebook kernel predating the expected-count record could have lost+ four legs and reported one, which against a default of one read as a pass.+ With the plan unknown a failure still fails, but a clean set is infra."""+ monkeypatch.setattr(launch, "fetch_evidence", lambda slug, dest, deadline = None: {})+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)+ api = _StubApi([], {_ENTRY["slug"]: "COMPLETE"})+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "infra" and record["expected"] is None, record+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": False, "payload": "x"}])+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)+ assert record["verdict"] == "fail"+ # Studio kernels carry exactly one report and keep the default.+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])+ studio = dict(_ENTRY, kind = "studio", slug = "me/unsloth-t4-ci-sabcdef01-1111")+ api = _StubApi([], {studio["slug"]: "COMPLETE"})+ assert (+ collect.collect_one(api, studio, tmp_path, expect = 1, max_age_hours = 3.0)["verdict"] == "pass"+ )+++def test_every_gate_budget_covers_the_reaper_window():+ """Nothing deletes the kernel now, so one that ignores its own timeout bills+ until the collector reaps it, up to a schedule interval and a job timeout+ late. The gate admits a run when `remaining >= budget + reserve`, so a+ budget below that window lets a wedged kernel bill into the reserve."""+ collect_wf = _wf(COLLECT_WF)+ job = collect_wf["jobs"]["collect"]+ on = collect_wf.get("on") or collect_wf.get(True) or {} # PyYAML reads a bare `on:` as True+ crons = [s["cron"] for s in on.get("schedule") or []]+ assert crons, "the collector is not scheduled"+ minutes = sorted(int(m) for m in crons[0].split()[0].split(","))+ interval_min = max(b - a for a, b in zip(minutes, minutes[1:])) if len(minutes) > 1 else 60+ source = COLLECT_WF.read_text(encoding = "utf-8")+ m = re.search(r"--max-age-hours '\$\{\{ inputs\.max_age_hours \|\| '(\d+)' \}\}'", source)+ assert m, "the collector's default age ceiling is not readable from the workflow"+ max_age = float(m.group(1))+ assert max_age == collect.DEFAULT_MAX_AGE_HOURS+ window = max_age + (interval_min + job["timeout-minutes"]) / 60+ for path in (NOTEBOOK_WF, STUDIO_WF):+ budgets = {+ int(b) for b in re.findall(r"--budget-hours (\d+)", path.read_text(encoding = "utf-8"))+ }+ assert len(budgets) == 1, (path.name, budgets)+ assert (+ budgets.pop() >= window+ ), f"{path.name}: a wedged kernel can bill {window:.2f}h before it is reaped"diff --git a/tests/kaggle/test_launch_cleanup.py b/tests/kaggle/test_launch_cleanup.pyindex f05411fb269..81ed9e7ade0 100644--- a/tests/kaggle/test_launch_cleanup.py+++ b/tests/kaggle/test_launch_cleanup.py@@ -227,6 +227,7 @@ def _push_ok(slug: str):kernel_timeout_sec,accelerator = "NvidiaTeslaT4",attempted = None,+ **kwargs,):attempted = [] if attempted is None else attemptedattempted.append(slug)@@ -481,7 +482,7 @@ def _waiting_launcher(outdir: Path) -> str:" launch._api = lambda *a, **k: _Api()"," launch.sweep_orphans = lambda *a, **k: []"," def _push(notebook, user, kernel_timeout_sec,",- " accelerator='NvidiaTeslaT4', attempted=None):",+ " accelerator='NvidiaTeslaT4', attempted=None, **kwargs):"," attempted = [] if attempted is None else attempted"," attempted.append('me/k-1')"," launch._inflight_add('me/k-1')",@@ -1063,7 +1064,7 @@ def test_a_kernel_pushed_before_the_signal_is_still_deleted(tmp_path):" launch.sweep_orphans = lambda *a, **k: []"," calls = []"," def _push(notebook, user, kernel_timeout_sec,",- " accelerator='NvidiaTeslaT4', attempted=None):",+ " accelerator='NvidiaTeslaT4', attempted=None, **kwargs):"," calls.append(notebook)"," attempted = [] if attempted is None else attempted"," slug = 'me/k-%d' % len(calls)",diff --git a/tests/kaggle/test_studio_gpu_harness.py b/tests/kaggle/test_studio_gpu_harness.pyindex f47095a7d15..fc8461a0e83 100644--- a/tests/kaggle/test_studio_gpu_harness.py+++ b/tests/kaggle/test_studio_gpu_harness.py@@ -950,16 +950,20 @@ def test_the_two_legs_together_fit_inside_the_ci_allowance():def test_the_reserve_leaves_ci_the_fifty_hours_it_is_allowed():source = WORKFLOW.read_text(encoding = "utf-8")assert "--reserve-hours 10" in source- assert "--budget-hours 2" in source+ assert "--budget-hours 4" in source+ assert "--budget-hours 2" not in source-def test_the_budget_hours_flag_covers_the_kernel_ceiling():- """budget-hours is the worst case one invocation can cost, and the- kernel ceiling is what enforces that worst case."""+def test_the_budget_hours_flag_covers_the_reaper_window():+ """budget-hours is the worst case one invocation can cost. Nothing waits on+ the kernel now, so a kernel that ignores its own timeout is left to the+ collector's reaper window rather than to the kernel ceiling."""source = WORKFLOW.read_text(encoding = "utf-8")assert "--kernel-timeout-sec 4200" in sourceceiling_hours = 4200 / 3600- assert ceiling_hours <= 2.0+ budgets = {int(b) for b in re.findall(r"--budget-hours (\d+)", source)}+ assert len(budgets) == 1, budgets+ assert budgets.pop() >= ceiling_hoursdef test_the_opt_in_label_the_summary_names_is_the_one_the_gate_reads():@@ -1671,6 +1675,44 @@ def test_a_nested_executed_notebook_is_read_too(tmp_path):assert total and len(chunks) == total+def test_evidence_that_is_json_but_not_a_notebook_does_not_raise(tmp_path):+ """`[]`, a string cell, a numeric output: all valid JSON, all seen back from+ a kernel. The reader used to raise on the first `.get`, failing the job+ after its verdict was already posted."""+ evidence = tmp_path / "evidence" / "unsloth-t4-ci-1234"+ evidence.mkdir(parents = True)+ (evidence / "a_output.ipynb").write_text("[]", encoding = "utf-8")+ (evidence / "b_output.ipynb").write_text(+ json.dumps({"cells": ["not a cell", {"outputs": [7, {"text": ["x", 1]}]}]}),+ encoding = "utf-8",+ )+ blob = _bundle({"studio.log": b"a log"})+ (evidence / "c_output.ipynb").write_text(+ json.dumps({"cells": [{"outputs": [{"text": "\n".join(_chunk_lines(blob))}]}]}),+ encoding = "utf-8",+ )+ streams = list(collect_evidence.iter_text(tmp_path / "evidence"))+ assert "x1" in streams+ chunks, total = collect_evidence.collect_chunks(iter(streams))+ assert total and len(chunks) == total+++def test_the_evidence_unpack_is_best_effort_on_the_gpu_job():+ """The notebook and scheduled unpacks already are; this one decided the+ colour of a job whose verdict had already been posted."""+ yaml = pytest.importorskip("yaml")+ workflow = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))+ steps = [+ s+ for job in workflow["jobs"].values()+ for s in job.get("steps", [])+ if "collect_evidence.py" in (s.get("run") or "")+ ]+ assert steps, "no unpack step"+ for step in steps:+ assert step.get("continue-on-error") is True, step.get("name")++# ------------------------------------------------------- a diverged training rundiff --git a/tests/kaggle/test_t4_ci_transport.py b/tests/kaggle/test_t4_ci_transport.pyindex 896d1895948..3e7cf25a374 100644--- a/tests/kaggle/test_t4_ci_transport.py+++ b/tests/kaggle/test_t4_ci_transport.py@@ -1171,6 +1171,7 @@ def _drive_main(kernel_timeout_sec,accelerator = "NvidiaTeslaT4",attempted = None,+ **kwargs,):clock["t"] += push_secondsoutcome = outcomes.pop(0)@@ -1634,6 +1635,7 @@ def _accepting_push(slug: str):kernel_timeout_sec,accelerator = "NvidiaTeslaT4",attempted = None,+ **kwargs,):if attempted is not None:attempted.append(slug)@@ -2330,7 +2332,7 @@ def test_main_bounds_the_whole_evidence_phase_it_is_budgeted_for(monkeypatch, tmmonkeypatch.setattr(launch,"push",- lambda nb, user, t, accelerator = "NvidiaTeslaT4", attempted = None: (+ lambda nb, user, t, accelerator = "NvidiaTeslaT4", attempted = None, **kwargs: (attempted.append(f"{user}/s{len(attempted)}"),{"ok": True, "slug": attempted[-1], "attempts": list(attempted)},)[1],diff --git a/tests/kaggle/test_t4_smoke_harness.py b/tests/kaggle/test_t4_smoke_harness.pyindex 91e4a42655f..5811d6bc9ae 100644--- a/tests/kaggle/test_t4_smoke_harness.py+++ b/tests/kaggle/test_t4_smoke_harness.py@@ -591,13 +591,23 @@ def test_the_gate_job_deadline_exceeds_the_gates_own_bound():"""import gate- worst = 3 * gate.SOCKET_TIMEOUT_SEC + gate.SURVEY_BUDGET_SEC- before_the_gate = 120- timeout_s = _workflow()["jobs"]["gate"]["timeout-minutes"] * 60- assert timeout_s >= worst + before_the_gate, (- f"the gate can take {worst}s, the steps before it up to "- f"{before_the_gate}s, and the job is killed at {timeout_s}s"+ # Per account: authentication, username and quota, each at the socket+ # ceiling. Then ONE survey budget shared across every account (two back to+ # back would be the bug), and the call still in flight when it expires.+ accounts = len(gate.DEFAULT_ACCOUNT_ENVS)+ worst = (+ accounts * 3 * gate.SOCKET_TIMEOUT_SEC + gate.SURVEY_BUDGET_SEC + gate.SOCKET_TIMEOUT_SEC)+ before_the_gate = 120+ for path in (WORKFLOW, WORKFLOW.parent / "kaggle-t4-studio-gpu-ci.yml"):+ workflow = pytest.importorskip("yaml").safe_load(path.read_text(encoding = "utf-8"))+ timeout_s = workflow["jobs"]["gate"]["timeout-minutes"] * 60+ assert timeout_s >= worst + before_the_gate, (+ f"{path.name}: the gate can take {worst}s, the steps before it up to "+ f"{before_the_gate}s, and the job is killed at {timeout_s}s"+ )+ source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")+ assert "survey_deadline - time.monotonic()" in source, "the surveys do not share one budget"def test_an_unsampled_invocation_is_a_skip_not_a_failure(monkeypatch, tmp_path):@@ -672,7 +682,7 @@ def _run_gate_against(monkeypatch.setattr(gate, "client_username", lambda api: "danielhanchen")monkeypatch.setattr(gate, "remaining_gpu_hours", lambda api: dict(quota))monkeypatch.setattr(- gate, "survey_kernels", answer if callable(answer) else (lambda api: answer)+ gate, "survey_kernels", answer if callable(answer) else (lambda api, **k: answer))code, outputs = _run_gate(monkeypatch, tmp_path, "--force", "true", *extra)summary_path = tmp_path / "summary.md"@@ -2732,15 +2742,20 @@ def test_a_dispatched_ref_is_resolved_to_one_commit():than a commit, so the drift is invisible afterwards too. Same hazard as thezoo pin below, same answer."""- steps = _workflow()["jobs"]["t4-smoke"]["steps"]+ workflow = _workflow()+ steps = workflow["jobs"]["t4-smoke"]["steps"]names = [s.get("name") for s in steps]ref = steps[names.index("Resolve the ref under test")]assert ref["env"]["UNSLOTH_REF"] == "${{ inputs.unsloth_ref }}"- # Resolved once, here, and retried before it gives up.- assert "git ls-remote https://github.com/unslothai/unsloth" in ref["run"]- assert "for attempt in 1 2 3" in ref["run"]+ # Resolved once, by the GATE, and retried there; this job takes that answer+ # rather than asking a moving ref again.+ gate_ref = next(s for s in workflow["jobs"]["gate"]["steps"] if s.get("id") == "ref")+ assert "git ls-remote https://github.com/unslothai/unsloth" in gate_ref["run"]+ assert "for attempt in 1 2 3" in gate_ref["run"]# A full commit needs no resolving and ls-remote would not answer for one.- assert "^[0-9a-f]{40}$" in ref["run"]+ assert "^[0-9a-f]{40}$" in gate_ref["run"]+ assert ref["env"]["GATE_SHA"] == "${{ needs.gate.outputs.head_sha }}"+ assert "$(git ls-remote" not in ref["run"]# A ref that cannot be pinned stands the run down rather than installing# a moving branch, exactly as an unresolvable zoo commit does.assert "stand_down=true" in ref["run"]@@ -2762,7 +2777,7 @@ def test_the_resolve_step_pins_every_shape_of_ref_it_can_be_given(tmp_path):thing, and this one has four branches: no input, a mutable branch or tag, acommit needing no resolution, and a ref that resolves to nothing."""- steps = _workflow()["jobs"]["t4-smoke"]["steps"]+ steps = _workflow()["jobs"]["gate"]["steps"]script = next(s for s in steps if s.get("id") == "ref")["run"]def drive(@@ -2794,25 +2809,69 @@ def test_the_resolve_step_pins_every_shape_of_ref_it_can_be_given(tmp_path):return dict(line.split("=", 1) for line in out.read_text().splitlines() if "=" in line)# No input: the head SHA under test, which is also the tree checked out.- assert drive("", "") == {"ref": "headsha"}+ assert drive("", "") == {"head_sha": "headsha"}# A branch resolves to the commit it points at, once, for all four legs.main_sha = "dead" + "0" * 36- assert drive("main", f"{main_sha}\trefs/heads/main\n") == {"ref": main_sha}+ assert drive("main", f"{main_sha}\trefs/heads/main\n") == {"head_sha": main_sha}# An annotated tag lists the tag object first; the COMMIT is what pip can# check out, and it is on the ^{} line.tag, commit = "aaaa" + "0" * 36, "bbbb" + "0" * 36assert drive("v1.2", f"{tag}\trefs/tags/v1.2\n{commit}\trefs/tags/v1.2^{{}}\n") == {- "ref": commit+ "head_sha": commit}# A full commit is already immutable, and ls-remote answers nothing for# one, so it must not be sent there at all.- assert drive("f" * 40, "") == {"ref": "f" * 40}- # Nothing resolved: stand down rather than install a moving branch.- assert drive("no-such-branch", "") == {"stand_down": "true"}+ assert drive("f" * 40, "") == {"head_sha": "f" * 40}+ # Nothing resolved: an EMPTY key, which the GPU job turns into a stand-down.+ assert drive("no-such-branch", "") == {"head_sha": ""}# The value is free text from a dispatch form and reaches no shell.- assert drive("'; touch pwned; echo '", "") == {"stand_down": "true"}+ assert drive("'; touch pwned; echo '", "") == {"head_sha": ""}assert not (tmp_path / "pwned").exists()+ # The GPU job's step takes the gate's answer and never resolves again.+ gpu_script = next(s for s in _workflow()["jobs"]["t4-smoke"]["steps"] if s.get("id") == "ref")[+ "run"+ ]++ def drive_gpu(+ unsloth_ref,+ gate_sha,+ fetch_exit = 0,+ ):+ work = tmp_path / f"gpu{abs(hash((unsloth_ref, gate_sha, fetch_exit)))}"+ stub = work / "bin"+ stub.mkdir(parents = True)+ (stub / "git").write_text(+ "#!/bin/sh\n"+ 'case "$1" in\n'+ ' ls-remote) echo "ls-remote must not be called" >&2; exit 97 ;;\n'+ ' fetch) exit "$GIT_FETCH_EXIT" ;;\n'+ " *) exit 0 ;;\n"+ "esac\n"+ )+ (stub / "sleep").write_text("#!/bin/sh\nexit 0\n")+ for name in ("git", "sleep"):+ (stub / name).chmod(0o755)+ out = work / "github_output"+ out.write_text("", encoding = "utf-8")+ env = dict(+ os.environ,+ PATH = f"{stub}:{os.environ['PATH']}",+ GITHUB_OUTPUT = str(out),+ UNSLOTH_REF = unsloth_ref,+ HEAD_SHA = "headsha",+ GATE_SHA = gate_sha,+ GIT_FETCH_EXIT = str(fetch_exit),+ )+ done = subprocess.run(["bash", "-c", gpu_script], env = env, capture_output = True, text = True)+ assert done.returncode == 0, done.stderr+ return dict(line.split("=", 1) for line in out.read_text().splitlines() if "=" in line)++ assert drive_gpu("", "") == {"ref": "headsha"}+ assert drive_gpu("main", main_sha) == {"ref": main_sha}+ assert drive_gpu("main", "") == {"stand_down": "true"}+ assert drive_gpu("main", main_sha, fetch_exit = 128) == {"stand_down": "true"}+def test_the_harness_stays_on_the_checked_out_tree_when_a_ref_is_dispatched():"""Resolving the INSTALL is not the same as checking that ref out.@@ -3039,10 +3098,11 @@ def test_the_account_is_rechecked_after_the_concurrency_slot_is_held():# --force skips the sampling draw only; the dice were rolled by the gate.assert "--force true" in recheck["run"]assert "--reserve-hours" in recheck["run"] and "--kernels" in recheck["run"]- # and it is the last thing before the push.- assert names[names.index("Recheck the Kaggle account") + 2] == "Launch on Kaggle and collect"- launch = steps[names.index("Launch on Kaggle and collect")]- assert launch["if"] == "steps.recheck.outputs.should_run == 'true'"+ # and nothing that spends a session runs unless it approved. On the+ # dispatch step's condition, not step order: collection now sits between the+ # two, so a positional rule would assert the file's shape, not the property.+ dispatch = steps[names.index("Dispatch to Kaggle")]+ assert "steps.recheck.outputs.should_run == 'true'" in dispatch["if"]def test_the_exhausted_quota_failure_reaches_the_pull_request():@@ -3196,8 +3256,8 @@ def test_an_unresolvable_zoo_commit_stands_the_run_down():# Nothing that spends a kernel runs after a stand-down.for name in ("Build the kernel notebooks", "Recheck the Kaggle account"):assert "steps.pins.outputs.stand_down != 'true'" in steps[names.index(name)]["if"]- launch_step = steps[names.index("Launch on Kaggle and collect")]- assert launch_step["if"] == "steps.recheck.outputs.should_run == 'true'"+ launch_step = steps[names.index("Dispatch to Kaggle")]+ assert "steps.recheck.outputs.should_run == 'true'" in launch_step["if"]def test_a_step_count_that_could_only_report_red_stands_the_run_down():@@ -3405,7 +3465,12 @@ def test_an_evidence_upload_outage_cannot_colour_the_check_red():assert "::warning" in warn["run"]# The verdict still runs, and still on its own condition.report = steps[names.index("Report")]- assert report["if"] == "always() && steps.recheck.outputs.should_run == 'true'"+ # always(), so a failed upload still reaches the verdict, and gated on "there+ # is evidence to read" rather than "the recheck approved": a dispatching run+ # approves but produces none, and a reporter over an empty directory prints+ # "0 of 5 payloads", which reads as failure rather than pending.+ assert report["if"].startswith("always()")+ assert "hashFiles('kaggle_evidence/**/*_output.ipynb')" in report["if"]assert "steps.evidence" not in report["if"]@@ -4602,7 +4667,8 @@ def test_a_dispatched_commit_is_proven_to_exist_before_the_quota_is_spent(tmp_paGITHUB_OUTPUT = str(out),UNSLOTH_REF = unsloth_ref,HEAD_SHA = "headsha",- LS_OUT = ls_remote,+ # What the gate resolved: the GPU step takes it as given.+ GATE_SHA = ls_remote.split("\t", 1)[0] if ls_remote else unsloth_ref,GIT_FETCH_EXIT = str(fetch_exit),FETCH_LOG = str(log),)diff --git a/tests/kaggle/test_two_account_selection.py b/tests/kaggle/test_two_account_selection.pyindex ce337a9dcda..c52e2a83f97 100644--- a/tests/kaggle/test_two_account_selection.py+++ b/tests/kaggle/test_two_account_selection.py@@ -283,8 +283,30 @@ def test_the_account_draw_is_salted_apart_from_the_sampling_draw():source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")picked = source.split("def weighted_pick", 1)[1].split("\ndef ", 1)[0]assert (- 'sha256(("account:" + run_id)' in picked- ), "the account draw hashes the bare run id, which is what sampled_in hashes"+ 'sha256(("account:" + key)' in picked+ ), "the account draw hashes the bare key, which is what sampled_in hashes"+++def test_every_run_of_one_commit_lands_on_the_same_account():+ """A label or a forced dispatch starts a second run of the SAME commit with+ a new run id. Keyed on the run id the draw could pick the other account,+ which cannot see the first's kernel, and dispatch a duplicate."""+ weights = {"1": 60.0, "2": 30.0}+ sha = "0123456789abcdef0123456789abcdef01234567"+ assert len({gate.weighted_pick(sha, weights)[0] for _ in range(5)}) == 1+ source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")+ assert (+ "account_key = (args.head_sha or" in source+ ), "the gate does not key the draw on the commit"+ for path in (NOTEBOOK_WF, STUDIO_WF):+ gate_steps = [+ s+ for _j, _n, s in _steps(_wf(path))+ if "gate.py" in (s.get("run") or "") and "--percent" in s["run"]+ ]+ assert gate_steps, f"{path.name} has no gate step"+ for step in gate_steps:+ assert "--head-sha" in step["run"], f"{path.name}: the gate is not told the commit"def test_an_account_with_no_readable_quota_gets_no_weight_but_keeps_its_turn():@@ -452,3 +474,251 @@ def test_neither_token_name_can_reach_the_kernel():assert (f'"{name}"' in source), f"{name} is not in the credential-leak guard, so a kernel could carry it"+++def test_a_kernel_already_running_this_commit_on_any_account_stands_the_run_down():+ """A handover lands a retry on the other account, whose GPU job collects+ with only its own token and sees nothing in flight. So the gate asks every+ account it surveys before it picks one."""+ sha = "abcdef0123456789" + "0" * 24+ own = [+ "danielhanchen/unsloth-t4-ci-sabcdef012345-1111 (RUNNING)", # studio, this commit+ "danielhanchen/unsloth-t4-ci-nffffffffffff-2222 (RUNNING)", # notebook, other commit+ "danielhanchen/unsloth-t4-ci-nabcdef01-3333 (QUEUED)", # notebook, old slug form+ ]+ assert gate.in_flight_for_commit(own, sha, "notebook") == (+ "danielhanchen/unsloth-t4-ci-nabcdef01-3333"+ )+ assert gate.in_flight_for_commit(own, sha, "studio") == (+ "danielhanchen/unsloth-t4-ci-sabcdef012345-1111"+ )+ assert gate.in_flight_for_commit(own, "1234567890ab", "notebook") is None+ assert gate.in_flight_for_commit(own, sha, "") is None+ assert gate.in_flight_for_commit(["someone/unsloth-probe-x (RUNNING)"], sha, "notebook") is None+ source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")+ main = source[source.index("def main(") :]+ # Every candidate is asked BEFORE any is chosen: the first loop over `order`+ # is the in-flight sweep, the selection loop comes after it.+ sweep, selection = main.split("for account_id in order:", 2)[1:]+ assert "in_flight_for_commit(survey" in sweep and "concurrency_verdict(" not in sweep+ assert "concurrency_verdict(" in selection and "in_flight_for_commit(" not in selection+ assert (+ "_survey(account_id)" in sweep and "_survey(account_id)" in selection+ ), "surveys are not shared between the sweep and the selection"+ for path, kind in ((NOTEBOOK_WF, "notebook"), (STUDIO_WF, "studio")):+ gate_steps = [+ s+ for _j, _n, s in _steps(_wf(path))+ if "gate.py" in (s.get("run") or "") and "--percent" in s["run"]+ ]+ assert gate_steps, path.name+ for step in gate_steps:+ assert f"--kind {kind}" in step["run"], f"{path.name}: the gate is not told its kind"+++def _drive_gate(+ monkeypatch,+ tmp_path,+ *,+ holder,+ outcomes = None,+ extra = (),+ clock = None,+ holder_slot = "1",+):+ """Run gate.main() with two stub accounts. `holder` is the account whose+ survey shows a notebook kernel of the commit under test; `outcomes` maps an+ account to a probe outcome other than ok, the client still handed back."""+ sha = "abcdef0123456789" + "0" * 24+ outcomes = outcomes or {}+ apis = {"1": object(), "2": object()}++ def probe(account_id, env_name, **_kw):+ quota = {"total_hours": 60.0 if account_id == "1" else 30.0, "remaining_hours": 50.0}+ record = {+ "account": account_id,+ "env": env_name,+ "outcome": outcomes.get(account_id, "ok"),+ "user": f"user{account_id}",+ "quota": quota,+ "total_hours": quota["total_hours"],+ "remaining_hours": 50.0,+ "reserve_hours": 1.0,+ }+ return record, apis[account_id]++ surveys_asked: list[tuple[str, float | None]] = []++ def survey(api, *a, **k):+ account_id = next(i for i, obj in apis.items() if obj is api)+ surveys_asked.append((account_id, k.get("budget_sec")))+ if clock is not None:+ clock[0] += 170.0 # a slow survey+ mark = "" if holder_slot == "1" else holder_slot+ own = (+ [f"user{holder}/unsloth-t4-ci-n{sha[:12]}-{mark}1111 (RUNNING)"]+ if account_id == holder+ else []+ )+ return {+ "busy": list(own),+ "own": own,+ "foreign": [],+ "complete": True,+ "out_of_budget": False,+ "surveyed": len(own),+ "unreadable": 0,+ "gone": 0,+ "window_hours": 12,+ }++ monkeypatch.setattr(gate, "probe_account", probe)+ monkeypatch.setattr(gate, "survey_kernels", survey)+ if clock is not None:+ monkeypatch.setattr(gate.time, "monotonic", lambda: clock[0])+ monkeypatch.setenv("KAGGLE_API_TOKEN", "x")+ monkeypatch.setenv("KAGGLE_API_TOKEN_2", "y")+ monkeypatch.setenv("GITHUB_OUTPUT", str(tmp_path / "out.txt"))+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "summary.md"))+ monkeypatch.setattr(+ sys,+ "argv",+ [+ "gate.py",+ "--budget-hours",+ "1",+ "--reserve-hours",+ "1",+ "--force",+ "true",+ "--head-sha",+ sha,+ "--kind",+ "notebook",+ "--run-id",+ "7",+ *extra,+ ],+ )+ code = gate.main()+ outputs = dict(+ line.split("=", 1)+ for line in (tmp_path / "out.txt").read_text().splitlines()+ if "=" in line+ )+ return code, outputs, surveys_asked, sha+++def _other(sha: str) -> tuple[str, str]:+ sampled = gate.weighted_pick(sha, {"1": 60.0, "2": 30.0})[0]+ return sampled, ("2" if sampled == "1" else "1")+++def test_the_gate_stands_down_when_the_other_account_already_runs_this_commit(+ monkeypatch, tmp_path+):+ """The sampled account is FREE and the other holds a kernel for this commit,+ which is what a retry after a handover looks like. A gate that only asks the+ account it is about to pick finds it clear and dispatches a duplicate."""+ _sampled, other = _other("abcdef0123456789" + "0" * 24)+ code, outputs, asked, _sha = _drive_gate(monkeypatch, tmp_path, holder = other)+ assert code == 0+ assert outputs["should_run"] == "false", outputs["reason"]+ assert f"account {other}" in outputs["reason"] and "already running" in outputs["reason"]+ assert {a for a, _b in asked} == {"1", "2"}, "the account holding the kernel was never asked"+ # One survey per account, reused by the selection loop.+ assert len(asked) == 2, asked+++def test_an_account_that_cannot_launch_is_still_asked_whether_it_runs_this_commit(+ monkeypatch, tmp_path+):+ """The account that dispatched this commit is the one likely to be short of+ quota now. Skipping it in the sweep hands the retry to the other account,+ which dispatches the same commit again."""+ _sampled, other = _other("abcdef0123456789" + "0" * 24)+ for outcome in ("insufficient_quota", "quota_unreadable"):+ code, outputs, asked, _sha = _drive_gate(+ monkeypatch, tmp_path, holder = other, outcomes = {other: outcome}+ )+ assert outputs["should_run"] == "false", (outcome, outputs["reason"])+ assert "already running" in outputs["reason"], (outcome, outputs["reason"])+ assert other in {a for a, _b in asked}, (outcome, asked)+++def test_the_second_slot_is_not_a_duplicate_but_its_own_retry_is(monkeypatch, tmp_path):+ """`slot: 2` is a second session on the same ref beside slot 1, so a slot-1+ kernel must not stand it down, while a retry of the slot-2 run itself must.+ The slot in the slug tells the two apart."""+ sampled, _unused = _other("abcdef0123456789" + "0" * 24)+ # Slot 1 kernel up, slot 2 asked for: runs.+ _c, outputs, _a, _s = _drive_gate(+ monkeypatch, tmp_path, holder = sampled, extra = ("--slot", "2"), holder_slot = "1"+ )+ assert outputs["should_run"] == "true", outputs["reason"]+ # Slot 2 kernel up, slot 2 asked for again: stands down.+ _c, outputs, _a, _s = _drive_gate(+ monkeypatch, tmp_path, holder = sampled, extra = ("--slot", "2"), holder_slot = "2"+ )+ assert outputs["should_run"] == "false", outputs["reason"]+ assert "slot 2" in outputs["reason"]+ # Slot 2 kernel up, slot 1 asked for: runs, it is the other seat.+ _c, outputs, _a, _s = _drive_gate(+ monkeypatch, tmp_path, holder = sampled, extra = ("--slot", "1"), holder_slot = "2"+ )+ assert outputs["should_run"] == "true", outputs["reason"]+ # And slot 1 against slot 1 still stands down.+ _c, outputs, _a, _s = _drive_gate(monkeypatch, tmp_path, holder = sampled, holder_slot = "1")+ assert outputs["should_run"] == "false", outputs["reason"]+ # The workflow threads its slot input through the gate, the collector's+ # in-flight check and the launcher (which writes it into the slug).+ steps = {s.get("id"): s for _j, _n, s in _steps(_wf(NOTEBOOK_WF)) if s.get("id")}+ for step_id, script in (+ ("decide", "gate.py"),+ ("collect", "collect.py"),+ ("launch", "launch.py"),+ ):+ run = steps[step_id]["run"]+ assert script in run+ assert "--slot '${{ inputs.slot || '1' }}'" in run, f"{step_id} is not told the slot"+ assert "--sha '${{ steps.ref.outputs.ref }}'" in steps["collect"]["run"]+++def test_the_surveys_share_one_budget(monkeypatch, tmp_path):+ """Two surveys each bounded by SURVEY_BUDGET_SEC outlive the gate job back+ to back. The second gets whatever the first left of ONE budget."""+ clock = [1000.0]+ _sampled, other = _other("abcdef0123456789" + "0" * 24)+ _code, _outputs, asked, _sha = _drive_gate(monkeypatch, tmp_path, holder = other, clock = clock)+ assert len(asked) == 2, asked+ first, second = asked[0][1], asked[1][1]+ assert first == pytest.approx(gate.SURVEY_BUDGET_SEC)+ assert second == pytest.approx(gate.SURVEY_BUDGET_SEC - 170.0), (first, second)+++def test_the_gate_is_keyed_on_the_commit_the_gpu_job_will_test():+ """A dispatch naming `unsloth_ref` tests that ref, not github.sha. Keyed on+ github.sha the gate looks for the wrong kernel, and once the default branch+ moves two runs of one ref can land on different accounts."""+ for path in (NOTEBOOK_WF, STUDIO_WF):+ gate_job = _wf(path)["jobs"]["gate"]+ steps = {s.get("id"): s for s in gate_job["steps"] if s.get("id")}+ ref = steps.get("ref")+ assert ref is not None, f"{path.name}: the gate job does not name the commit under test"+ assert ref["env"]["UNSLOTH_REF"] == "${{ inputs.unsloth_ref }}"+ assert "head_sha=" in ref["run"] and "git ls-remote" in ref["run"]+ decide = steps["decide"]+ assert (+ "--head-sha '${{ steps.ref.outputs.head_sha }}'" in decide["run"]+ ), f"{path.name}: the gate is keyed on a commit the GPU job may not test"+ assert "github.sha" not in decide["run"]+ # And the GPU job takes the gate's answer rather than resolving a+ # moving ref a second time.+ assert gate_job["outputs"]["head_sha"] == "${{ steps.ref.outputs.head_sha }}"+ gpu = next(+ s+ for _j, _n, s in _steps(_wf(path))+ if s.get("id") == "ref" and "GATE_SHA" in (s.get("env") or {})+ )+ assert gpu["env"]["GATE_SHA"] == "${{ needs.gate.outputs.head_sha }}"+ assert "$(git ls-remote" not in gpu["run"]