Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kaggle CI: dispatch the kernel and collect it later, instead of holding a runner (#10183)

Daniel Han6bc095ae0846
diff --git a/.github/scripts/kaggle_studio_ci/collect_evidence.py b/.github/scripts/kaggle_studio_ci/collect_evidence.py
index 20525e85fdc..a798983f5c9 100644
--- a/.github/scripts/kaggle_studio_ci/collect_evidence.py
+++ b/.github/scripts/kaggle_studio_ci/collect_evidence.py
@@ -58,11 +58,19 @@ def iter_text(path: Path):
nb = json.loads(nb_path.read_text(encoding = "utf-8", errors = "replace"))
except Exception: # noqa: BLE001
continue
- for cell in nb.get("cells", []):
- for output in cell.get("outputs", []):
+ # Kernels have returned valid JSON that is not a notebook (`[]`, or a
+ # non-object cell); raising here fails the job after the verdict posted.
+ if not isinstance(nb, dict):
+ continue
+ for cell in nb.get("cells") or []:
+ if not isinstance(cell, dict):
+ continue
+ for output in cell.get("outputs") or []:
+ if not isinstance(output, dict):
+ continue
text = output.get("text") or ""
if isinstance(text, list):
- text = "".join(text)
+ text = "".join(str(t) for t in text)
yield text
for log_path in sorted(path.rglob("kernel.log")):
raw = log_path.read_text(encoding = "utf-8", errors = "replace")
diff --git a/.github/scripts/kaggle_t4_ci/BUDGET.md b/.github/scripts/kaggle_t4_ci/BUDGET.md
index 0717c1bef07..f7c6c174521 100644
--- a/.github/scripts/kaggle_t4_ci/BUDGET.md
+++ b/.github/scripts/kaggle_t4_ci/BUDGET.md
@@ -146,6 +146,14 @@ reserves against its own budget rather than this one.
from `launch.py` and `--kernels`; do not edit the number here or in the
workflow without changing what it is derived from.
+The Studio leg's `--budget-hours` covers the reaper window, not its 70-minute
+kernel ceiling: nothing waits on the kernel now, and Kaggle has been caught
+leaving one RUNNING two hours past its own timeout. The collector reaps at
+`DEFAULT_MAX_AGE_HOURS` (3h) plus its 10-minute schedule and 30-minute job
+timeout, so 3.67h is the worst case one dispatch can bill and the budget is 4,
+not 2. `test_every_gate_budget_covers_the_reaper_window` recomputes it from
+the collector workflow for both legs.
+
## What would change these numbers
* A second Kaggle account doubles supply and the percentage could roughly
diff --git a/.github/scripts/kaggle_t4_ci/build_kernel.py b/.github/scripts/kaggle_t4_ci/build_kernel.py
index c17ee5daff3..9d55f0e8163 100644
--- a/.github/scripts/kaggle_t4_ci/build_kernel.py
+++ b/.github/scripts/kaggle_t4_ci/build_kernel.py
@@ -589,9 +589,14 @@ def build_driver(
shared_wheel_specs: tuple[str, ...] = (),
overlays: dict[str, tuple[str, ...]] | None = None,
all_card: tuple[str, ...] = (),
+ expected_reports: int | None = None,
) -> dict:
"""Kernel notebook that runs the payloads across the session's GPUs.
+ ``expected_reports`` is how many T4_SMOKE_REPORT lines to expect, echoed
+ into the kernel log so collect.py can judge the kernel days later. Defaults
+ to one per payload, less the report-less Studio install half.
+
``isolation`` maps a payload to whether its virtualenv may see the Kaggle
image's site-packages. Per payload, not per kernel, because legs sharing a
kernel do not share an answer: see ``Leg.system_site_packages``.
@@ -649,6 +654,8 @@ def build_driver(
_prefetch_builder().prefetch_cell(list(prefetch_repos)).encode("utf-8")
)
+ if expected_reports is None:
+ expected_reports = len(payloads) - (1 if cpu_lane else 0)
setup = f"""import base64, gzip, json, os, pathlib, subprocess, sys, threading, time
print("{DRIVER_SENTINEL} start", flush=True)
@@ -811,6 +818,7 @@ if N_GPU < EXPECTED_GPUS:
for name, blob in PAYLOADS.items():
(WORK / name).write_bytes(gzip.decompress(base64.b64decode(blob)))
print("{DRIVER_SENTINEL}_PAYLOADS " + json.dumps(sorted(PAYLOADS)), flush=True)
+print("{DRIVER_SENTINEL}_EXPECT " + json.dumps({{"reports": {expected_reports}}}), flush=True)
"""
runner = f'''results = {{}}
diff --git a/.github/scripts/kaggle_t4_ci/collect.py b/.github/scripts/kaggle_t4_ci/collect.py
new file mode 100644
index 00000000000..8edbe9c2a1e
--- /dev/null
+++ b/.github/scripts/kaggle_t4_ci/collect.py
@@ -0,0 +1,746 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Collect the kernels an earlier job dispatched and left running.
+
+``launch.py --dispatch`` pushes a kernel and exits within minutes instead of
+holding a runner for the forty Kaggle takes. That trade is only sound if
+something else finishes the job, and this is that something.
+
+THE ACCOUNT IS THE QUEUE. No database, artifact or branch sits between the
+dispatching job and this one, because each is a second place for the truth to
+go stale. ``kernels list --mine`` enumerates the kernels and the SLUG carries
+the commit and workflow (``launch.slug_name``), so this script rediscovers
+everything from the account and can run from any job, in any order, any number
+of times.
+
+Three outcomes per kernel, and they are not symmetric:
+
+* **terminal** -- download the evidence, judge the reports, DELETE, and emit a
+ commit status for the sha in the slug. The only path that produces a verdict.
+* **still running, within its age ceiling** -- left completely alone; a kernel
+ doing its job is not a problem to report.
+* **still running, past the ceiling** -- deleted, and reported as a failure to
+ collect rather than of the code. This is the reaper, and why the script runs
+ on a SCHEDULE: a dispatched kernel bills accelerator quota to its ceiling and
+ a quiet repo would leave it there.
+
+This script never talks to GitHub. It emits the statuses it wants posted into
+``collect_result.json`` and the workflow posts them (``post_statuses.py``): a
+process holding both a Kaggle and a GitHub credential is one bug away from
+sending one to the other.
+
+DELETION COMES LAST, after the status is delivered: ``--no-delete``, post, then
+``--delete-collected``. Deleting first would turn a transient API failure into
+the permanent loss of a verdict.
+
+SAFETY above all. This enumerates a whole ACCOUNT shared with a human and
+DELETES what it collects, so nothing is touched unless ``launch.parse_slug``
+recognises its name. An unrecognised kernel is invisible, not warned about.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import socket
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import launch # noqa: E402
+from gate import GONE_MARKERS, _as_naive_utc # noqa: E402
+
+# How long a dispatched kernel may stay in flight before the reaper takes it.
+# The ceiling is ours because Kaggle's has been caught failing: a kernel pushed
+# with `-t 5400` whose nbconvert crashed sat RUNNING two hours past it.
+#
+# Generous rather than tight: the wired notebook kernel measured 2101.8s, the
+# two-account run 41.5 min of job wall clock, and queueing behind another of
+# ours adds most of that again. Reaping a kernel that would have finished costs
+# a whole run and reports a failure nobody can act on.
+DEFAULT_MAX_AGE_HOURS = 3.0
+
+# Paging for the account walk. Not bounded by the reap ceiling (a kernel far
+# past it is what the reaper exists for), nor by a small page count: the
+# account is SHARED and human kernels are never deleted, so once enough newer
+# records sit above an uncollected kernel of ours a fixed cap would never reach
+# it again. The walk stops at a page entirely older than LISTING_HORIZON_HOURS
+# with none of ours on it, at the pass deadline, or at MAX_PAGES.
+PAGE_SIZE = 100
+MAX_PAGES = 50
+# Kaggle kills a session at 12h whatever else fails, so a kernel of ours that
+# last ran before this horizon is neither running nor owed a timely status.
+LISTING_HORIZON_HOURS = 24.0 * 7
+
+# The driver's own record of how many payload reports the kernel was BUILT to
+# produce, and the only one that survives dispatch: judging a five-payload
+# kernel against `--expect 1` turns a run that lost four of them into a pass.
+EXPECT_SENTINEL = "KAGGLE_T4_CI_DRIVER_EXPECT "
+
+# Wall clock for the whole collection (evidence downloads dominate). It runs on
+# a schedule, so a kernel missed this pass is reached five minutes later.
+BUDGET_SEC = 900
+
+# Per-kernel evidence budget, well under the total, so one slow download cannot
+# consume the pass and starve every kernel behind it.
+EVIDENCE_BUDGET_SEC = 300
+
+SOCKET_TIMEOUT_SEC = 120
+
+# The commit status context per workflow kind. A PUBLIC interface: branch
+# protection is configured against these strings, so renaming one silently
+# stops requiring the check it names.
+STATUS_CONTEXTS = {
+ "notebook": "kaggle-t4-notebook",
+ "studio": "kaggle-studio-gpu",
+}
+
+
+def _log(msg: str) -> None:
+ print(f"[collect] {msg}", flush = True)
+
+
+def _out(key: str, value: str) -> None:
+ path = os.environ.get("GITHUB_OUTPUT")
+ if not path:
+ return
+ try:
+ with open(path, "a", encoding = "utf-8") as fh:
+ fh.write(f"{key}={value}\n")
+ except OSError:
+ pass
+
+
+def kernel_age_hours(kernel, now: datetime) -> float | None:
+ """How long ago this kernel last started, or None if Kaggle did not say.
+
+ For an UNFINISHED kernel ``last_run_time`` is when the run started, which is
+ what the reap ceiling measures against. None rather than a default, treated
+ as too young to reap: guessing old deletes a live session.
+ """
+ last_run = _as_naive_utc(getattr(kernel, "last_run_time", None))
+ if last_run is None:
+ return None
+ return (now - last_run).total_seconds() / 3600.0
+
+
+def find_ours(
+ api,
+ now: datetime | None = None,
+ max_age_hours: float = DEFAULT_MAX_AGE_HOURS,
+ page_size: int = PAGE_SIZE,
+ max_pages: int = MAX_PAGES,
+ horizon_hours: float = LISTING_HORIZON_HOURS,
+ deadline: float | None = None,
+) -> list[dict]:
+ """Every kernel on this account that WE pushed, newest first.
+
+ The filter is ``launch.parse_slug`` and nothing else: an unrecognised kernel
+ is not ours to read, let alone delete. ``max_age_hours`` is the reap ceiling
+ the caller applies later, not a listing cutoff, because the kernel that most
+ needs collecting is the one a delayed pass finds hours past it. The walk
+ ends at the first page entirely older than ``horizon_hours`` carrying none
+ of ours, at ``deadline``, or at ``max_pages``.
+ """
+ now = now or datetime.now(timezone.utc).replace(tzinfo = None)
+ found: list[dict] = []
+ for page in range(1, max_pages + 1):
+ if deadline is not None and time.time() >= deadline:
+ _log(f"listing stopped at page {page}: out of budget")
+ break
+ kernels = (
+ api.kernels_list(mine = True, page = page, page_size = page_size, sort_by = "dateRun") or []
+ )
+ if not kernels:
+ break
+ ours_on_page = 0
+ beyond_horizon = 0
+ for kernel in kernels:
+ age = kernel_age_hours(kernel, now)
+ if age is not None and age > horizon_hours:
+ beyond_horizon += 1
+ ref = getattr(kernel, "ref", None)
+ if not ref:
+ continue
+ parsed = launch.parse_slug(ref)
+ if parsed is None:
+ continue
+ ours_on_page += 1
+ found.append(
+ {
+ "slug": ref,
+ "sha": parsed["sha"],
+ "kind": parsed["kind"],
+ "slot": parsed.get("slot", "1"),
+ "legacy": parsed["legacy"],
+ "age_hours": age,
+ }
+ )
+ if ours_on_page == 0 and beyond_horizon == len(kernels):
+ break
+ return found
+
+
+def _status_of(api, slug: str) -> str:
+ """The kernel's state, with a deleted kernel distinguished from an unknown one.
+
+ A kernel deleted moments ago still lists but 404s its status call:
+ definitively not running, not unreadable. Conflating the two makes the
+ reaper attack gone kernels and read a genuine outage as idle.
+ """
+ try:
+ raw = str(getattr(api.kernels_status(slug), "status", ""))
+ except Exception as exc: # noqa: BLE001
+ text = f"{exc}".lower()
+ if any(m in text for m in GONE_MARKERS):
+ return "GONE"
+ _log(f"status unreadable for {slug}: {type(exc).__name__}")
+ return "UNKNOWN"
+ match = launch._STATUS_RE.search(raw)
+ return match.group("status") if match else (raw.strip().upper() or "UNKNOWN")
+
+
+def verdict_of(reports: list[dict], expect: int) -> tuple[str, str]:
+ """Turn the payload reports into a verdict, exactly as launch.py does inline.
+
+ Kept identical so a dispatched run and a waited-on run reach the same
+ conclusion from the same evidence.
+ """
+ if not reports:
+ return "infra", (
+ "the kernel finished but produced no payload report; "
+ "nothing was learned about the code under test"
+ )
+ failing = [r for r in reports if not r.get("passed")]
+ if failing:
+ names = ", ".join(str(r.get("payload") or "?") for r in failing)
+ return (
+ "fail",
+ f"{len(failing)} of {len(reports)} payload(s) failed their assertions: {names}",
+ )
+ if len(reports) < expect:
+ return "partial", (
+ f"only {len(reports)} of {expect} payload(s) reported back, "
+ "so this is half a comparison rather than a result"
+ )
+ return "pass", f"all {len(reports)} payload(s) passed"
+
+
+def _gone(exc: BaseException) -> bool:
+ """Did this failure say the kernel is no longer there?
+
+ Three unserialised collectors run, so another can delete a kernel between
+ this pass seeing it terminal and downloading it. Not an infra failure: a
+ result somebody else already posted.
+ """
+ text = f"{exc}".lower()
+ return any(m in text for m in GONE_MARKERS)
+
+
+def _evidence_lines(dest: Path):
+ """Every line of driver output in the collected evidence, notebooks first."""
+ for nb_path in sorted(dest.rglob(f"*{launch.OUTPUT_SUFFIX}")):
+ try:
+ nb = json.loads(nb_path.read_text(encoding = "utf-8", errors = "replace"))
+ except Exception: # noqa: BLE001
+ continue
+ # Valid JSON that is not a notebook is skipped, not raised on: this
+ # runs outside the report guard, and an exception here writes no result
+ # file and wedges every later pass on this kernel.
+ if not isinstance(nb, dict):
+ continue
+ for cell in nb.get("cells") or []:
+ if not isinstance(cell, dict):
+ continue
+ for output in cell.get("outputs") or []:
+ if not isinstance(output, dict):
+ continue
+ text = output.get("text") or ""
+ if isinstance(text, list):
+ text = "".join(str(t) for t in text)
+ if isinstance(text, str):
+ yield from text.splitlines()
+ for log_path in sorted(dest.rglob("kernel.log")):
+ raw = log_path.read_text(encoding = "utf-8", errors = "replace")
+ yield from launch.flatten_kernel_log(raw).splitlines()
+
+
+def expected_reports(
+ dest: Path,
+ default: int,
+ kind: str = "",
+) -> int | None:
+ """How many payload reports this kernel was built to produce.
+
+ Read off the kernel's own evidence, the only record that survives dispatch.
+ A kernel predating the sentinel, or a Studio kernel (always one report),
+ answers ``default``.
+ """
+ for line in _evidence_lines(dest):
+ if not line.startswith(EXPECT_SENTINEL):
+ continue
+ try:
+ parsed = json.loads(line[len(EXPECT_SENTINEL) :].strip())
+ except ValueError:
+ continue
+ reports = parsed.get("reports") if isinstance(parsed, dict) else None
+ if isinstance(reports, int) and reports > 0:
+ return reports
+ # Only a notebook kernel's own record says how many payloads it carries.
+ # Without it the plan is UNKNOWN, not the caller's default, or a kernel that
+ # lost four legs reads as a pass. Studio kernels always carry exactly one.
+ if kind == "notebook":
+ return None
+ return default
+
+
+# How a verdict is reported to GitHub. `infra` and `partial` are deliberately
+# NOT failures: nothing was learned about the code, and a red the author cannot
+# act on is how a required check gets dropped from branch protection.
+VERDICT_STATE = {
+ "pass": "success",
+ "fail": "failure",
+ "partial": "success",
+ "infra": "success",
+ "reaped": "failure",
+}
+
+
+def collect_one(
+ api,
+ entry: dict,
+ outdir: Path,
+ expect: int,
+ max_age_hours: float,
+ delete: bool = True,
+ deadline: float | None = None,
+) -> dict:
+ """Read one kernel to a conclusion. Returns the record for the result file.
+
+ ``deadline`` is the whole pass's; the evidence download gets its own budget
+ clamped to what is left, so the last kernel cannot overrun the job.
+ """
+ slug = entry["slug"]
+ state = _status_of(api, slug)
+ record = {
+ **entry,
+ "state": state,
+ "verdict": None,
+ "reason": "",
+ "deleted": False,
+ "reports": 0,
+ }
+
+ if state in ("QUEUED", "RUNNING"):
+ age = entry.get("age_hours")
+ if age is not None and age > max_age_hours:
+ # Reported as a FAILURE of collection rather than dropped: it has
+ # been billing and will never produce a result, and silence leaves
+ # the commit reading "not run" forever.
+ record["verdict"] = "reaped"
+ # "released", not "deleted": the delete happens in a later step
+ # that can be refused, and this reason is posted before it runs.
+ record["reason"] = (
+ f"the kernel was still {state} after {age:.1f}h, past the {max_age_hours}h "
+ "ceiling, so it is released for deletion. It was billing accelerator "
+ "quota and would not have produced a result"
+ )
+ if delete:
+ record["deleted"] = launch.delete_kernel(slug, deadline = deadline)
+ if record["deleted"]:
+ record["reason"] = record["reason"].replace(
+ "is released for deletion", "was deleted"
+ )
+ _log(f"reaped {slug} ({state}, {age:.1f}h)")
+ return record
+ record["verdict"] = "pending"
+ record["reason"] = f"still {state}"
+ _log(f"pending {slug} ({state}, age {age if age is None else round(age, 2)}h)")
+ return record
+
+ if state == "UNKNOWN":
+ # Left alone: an unreadable status says nothing, and both actions here
+ # are destructive. The next pass asks again.
+ record["verdict"] = "pending"
+ record["reason"] = "status unreadable this pass"
+ return record
+
+ if state == "GONE":
+ record["verdict"] = "gone"
+ record["reason"] = "the kernel no longer exists; nothing to collect"
+ return record
+
+ if state not in launch.TERMINAL_OK | launch.TERMINAL_BAD:
+ # Only a KNOWN terminal state may download, judge, post and delete.
+ # Kaggle's enum can grow, and judging an unknown state posts a green
+ # `infra` for a run still to come and then deletes it.
+ record["verdict"] = "pending"
+ record["reason"] = f"state {state} is not one this collector judges; asked again next pass"
+ _log(f"unrecognised state {state} for {slug}; kept")
+ return record
+
+ # Terminal. Evidence FIRST, delete second: a delete before the download
+ # turns a finished run into a result nobody can read.
+ dest = outdir / slug.rsplit("/", 1)[-1]
+ try:
+ evidence_deadline = time.time() + EVIDENCE_BUDGET_SEC
+ if deadline is not None:
+ evidence_deadline = min(evidence_deadline, deadline)
+ evidence = launch.fetch_evidence(slug, dest, deadline = evidence_deadline)
+ except Exception as exc: # noqa: BLE001
+ record["evidence"] = None
+ if _gone(exc):
+ # Deleted by another pass between our status call and this
+ # download: their verdict stands, ours would be an `infra` on top.
+ record["verdict"] = "gone"
+ record["reason"] = "another collector finished this kernel first"
+ _log(f"{slug} was collected by another pass")
+ return record
+ # `pending` rather than `infra`: `infra` posts green and releases the
+ # kernel, losing a real result to one transient failure.
+ record["verdict"] = "pending"
+ record["reason"] = (
+ f"the kernel finished but its evidence would not download this pass "
+ f"({type(exc).__name__}); kept for the next one"
+ )
+ _log(f"could not collect {slug}: {type(exc).__name__}")
+ return record
+
+ record["evidence"] = evidence
+ if evidence.get("truncated"):
+ # fetch_evidence flags a spent budget instead of raising, so same
+ # answer as above: judging a short set reads a run that lost half its
+ # notebooks as whatever the surviving half says.
+ record["verdict"] = "pending"
+ record["reason"] = (
+ "the evidence download was incomplete this pass; the kernel is kept for the next one"
+ )
+ _log(f"incomplete evidence for {slug}; left for the next pass")
+ return record
+
+ try:
+ reports = launch.extract_reports(dest)
+ except Exception as exc: # noqa: BLE001
+ # Evidence that downloaded but cannot be read will not read better next
+ # pass, and raising here wedges every later one. Nothing was learned, so
+ # `infra`, and the kernel is released.
+ reports = []
+ record["report_error"] = f"{type(exc).__name__}: {exc}"[:200]
+ _log(f"unreadable report in {slug}: {type(exc).__name__}")
+ record["reports"] = len(reports)
+ expect = expected_reports(dest, expect, record.get("kind") or "")
+ record["expected"] = expect
+ if expect is None:
+ # Plan unknown: a failure is still a failure, but completeness cannot
+ # be shown, so no pass is claimed.
+ verdict, reason = verdict_of(reports, len(reports))
+ if verdict == "pass":
+ verdict, reason = (
+ "infra",
+ (
+ f"{len(reports)} payload report(s) read, but the kernel predates the "
+ "record of how many it was built to produce, so completeness cannot be "
+ "judged and no pass is claimed"
+ ),
+ )
+ else:
+ verdict, reason = verdict_of(reports, expect)
+ if record.get("report_error") and verdict == "infra":
+ reason = (
+ f"the evidence downloaded but its report could not be read ({record['report_error']})"
+ )
+ if state in launch.TERMINAL_BAD and verdict == "infra":
+ reason = (
+ f"the kernel ended {state} without reporting; the session died rather than the code"
+ )
+ record["verdict"] = verdict
+ record["reason"] = reason
+ if delete:
+ record["deleted"] = launch.delete_kernel(slug, deadline = deadline)
+ if not record["deleted"]:
+ _log(f"could not delete {slug}; it may keep billing")
+ _log(f"collected {slug}: {verdict} ({reason})")
+ return record
+
+
+def statuses_from(records: list[dict], target_url: str = "") -> list[dict]:
+ """The commit statuses to post, one per collected kernel that names a commit.
+
+ THIS is the check that means "the GPU tests passed": the dispatching job
+ succeeds by dispatching, so branch protection pointing at it would require
+ only that Kaggle accepted a push. A legacy slug carries no sha to attribute
+ to and yields nothing.
+ """
+ out: dict[tuple[str, str], dict] = {}
+ for record in records:
+ if record.get("verdict") in (None, "pending", "gone"):
+ continue
+ sha = record.get("sha")
+ kind = record.get("kind")
+ if not sha or kind not in STATUS_CONTEXTS:
+ continue
+ context = STATUS_CONTEXTS[kind]
+ state = VERDICT_STATE.get(record["verdict"], "success")
+ # One line: GitHub shows the description on one line and caps it at 140.
+ description = " ".join(f"{record['verdict']}: {record['reason']}".split())[:140]
+ status = {
+ "sha": sha,
+ "context": context,
+ "state": state,
+ "description": description,
+ "target_url": target_url,
+ "slug": record["slug"],
+ "slugs": [record["slug"]],
+ }
+ # Two kernels for one commit under one context (slots 1 and 2) must not
+ # race to post last: a failure wins whichever kernel found it, and both
+ # are named so the delete step releases both. Keyed on the sha as the
+ # slug carries it; only the commits API can say whether an 8 and a 12
+ # character slug name one commit, so post_statuses merges after
+ # resolving.
+ key = (sha, context)
+ prior = out.get(key)
+ if prior is None:
+ out[key] = status
+ continue
+ prior["slugs"].append(record["slug"])
+ if state == "failure" and prior["state"] != "failure":
+ status["slugs"] = prior["slugs"]
+ out[key] = status
+ return list(out.values())
+
+
+# Verdicts whose kernel is finished with once the status is delivered. `pending`
+# and `gone` own nothing to delete; `None` never got as far as a verdict.
+DELETABLE = {"pass", "fail", "partial", "infra", "reaped"}
+
+# Ceiling on the release phase. delete_kernel allows three 180-second attempts
+# per kernel, so an unbounded release after a full BUDGET_SEC collection could
+# outlive the job. Leftovers are released by the next pass.
+RELEASE_BUDGET_SEC = 600
+
+
+def delete_collected(result_path: Path, posted_path: Path | None) -> int:
+ """Release the kernels a ``--no-delete`` pass judged, now that their
+ statuses are posted.
+
+ ``posted.json`` names the slugs delivered, refused, and whose commit is
+ gone. A refused post KEEPS its kernel, the only retry there is. A gone
+ commit releases it; holding it only bills.
+ """
+ data = json.loads(result_path.read_text(encoding = "utf-8"))
+ posted = (
+ json.loads(posted_path.read_text(encoding = "utf-8"))
+ if posted_path and posted_path.exists()
+ else {}
+ )
+ # Refused and rejected alike: nothing reached GitHub, so the kernel stays.
+ failed = set(posted.get("failed") or []) | set(posted.get("invalid") or [])
+ if posted_path and not posted_path.exists():
+ # The poster never ran. Keep every kernel that had something to post,
+ # or their verdicts are lost.
+ failed = {s["slug"] for s in data.get("statuses", [])} | {
+ slug for s in data.get("statuses", []) for slug in s.get("slugs", [])
+ }
+ outcome = {"deleted": [], "kept": [], "failed_delete": []}
+ deadline = time.time() + RELEASE_BUDGET_SEC
+ for record in data.get("kernels") or []:
+ slug = record.get("slug")
+ if not slug or record.get("verdict") not in DELETABLE:
+ continue
+ if record.get("deleted"):
+ continue
+ if slug in failed:
+ outcome["kept"].append(slug)
+ _log(f"kept {slug}: its status did not post; the next pass retries")
+ continue
+ if time.time() >= deadline:
+ outcome["kept"].append(slug)
+ _log(f"kept {slug}: release budget spent; the next pass releases it")
+ continue
+ if launch.delete_kernel(slug, deadline = deadline):
+ outcome["deleted"].append(slug)
+ _log(f"deleted {slug}")
+ else:
+ outcome["failed_delete"].append(slug)
+ print(f"::warning title=Kaggle kernel not deleted::{slug} may keep billing")
+ result_path.with_name("delete_result.json").write_text(
+ json.dumps(outcome, indent = 2), encoding = "utf-8"
+ )
+ return 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--outdir", required = True, help = "where evidence is downloaded")
+ ap.add_argument(
+ "--expect",
+ type = int,
+ default = 1,
+ help = "payload reports a collected kernel should carry",
+ )
+ ap.add_argument("--max-age-hours", type = float, default = DEFAULT_MAX_AGE_HOURS)
+ ap.add_argument(
+ "--sha",
+ default = "",
+ help = "when given, report whether a kernel for THIS commit is already in "
+ "flight, so the caller can skip dispatching a second one",
+ )
+ ap.add_argument(
+ "--kind",
+ default = "",
+ choices = ("", *launch.KIND_CODES),
+ help = "narrow --sha to one workflow's kernels",
+ )
+ ap.add_argument(
+ "--slot",
+ default = "1",
+ help = "narrow --sha to one session slot's kernels: slot 2 runs beside slot 1 on "
+ "the same commit by design, and only its own retry is a duplicate",
+ )
+ ap.add_argument("--target-url", default = "", help = "run URL to attach to each status")
+ ap.add_argument(
+ "--no-delete",
+ action = "store_true",
+ help = "collect and report without deleting. The workflows always pass "
+ "this and delete afterwards with --delete-collected, once the statuses "
+ "are posted; a kernel left up after that keeps billing",
+ )
+ ap.add_argument(
+ "--require-auth",
+ action = "store_true",
+ help = "fail rather than skip when Kaggle authentication fails. The "
+ "scheduled reaper passes this: it runs with the repository's own "
+ "secrets, so an auth failure there is a broken token, not a fork",
+ )
+ ap.add_argument(
+ "--delete-collected",
+ default = "",
+ metavar = "COLLECT_RESULT_JSON",
+ help = "delete the kernels a previous --no-delete pass collected, except "
+ "those whose status did not post (see --posted). Nothing else runs",
+ )
+ ap.add_argument(
+ "--posted",
+ default = "",
+ metavar = "POSTED_JSON",
+ help = "post_statuses.py's record of which statuses were delivered",
+ )
+ args = ap.parse_args()
+
+ if args.delete_collected:
+ return delete_collected(
+ Path(args.delete_collected), Path(args.posted) if args.posted else None
+ )
+
+ socket.setdefaulttimeout(SOCKET_TIMEOUT_SEC)
+ outdir = Path(args.outdir)
+ outdir.mkdir(parents = True, exist_ok = True)
+ result: dict = {"owner": None, "kernels": [], "statuses": [], "in_flight_for_sha": False}
+
+ def finish(code: int = 0) -> int:
+ (outdir / "collect_result.json").write_text(json.dumps(result, indent = 2), encoding = "utf-8")
+ _out("in_flight", "true" if result["in_flight_for_sha"] else "false")
+ _out(
+ "collected",
+ str(
+ sum(
+ 1
+ for k in result["kernels"]
+ if k.get("verdict") not in (None, "pending", "gone")
+ )
+ ),
+ )
+ _out("pending", str(sum(1 for k in result["kernels"] if k.get("verdict") == "pending")))
+ return code
+
+ if args.require_auth and not os.environ.get("KAGGLE_API_TOKEN"):
+ # Not configured is not broken: the collector's matrix is static, so a
+ # repository with one Kaggle account leaves the second secret empty.
+ # Warn and pass. A token that is present and refused is still red.
+ print(
+ "::warning title=Kaggle account not configured::KAGGLE_API_TOKEN is empty in "
+ "this job, so nothing was collected for this account. A repository with one "
+ "Kaggle account leaves the second secret unset; if this account is meant to "
+ "be configured, set its secret."
+ )
+ _log("no token in this job; nothing collected")
+ return finish()
+
+ try:
+ api = launch._api()
+ except BaseException as exc: # noqa: BLE001
+ if isinstance(exc, KeyboardInterrupt):
+ raise
+ if args.require_auth:
+ # The reaper runs on the repository's own token, so this is an
+ # expired credential. A green empty pass would hide it while kernels
+ # bill to their ceiling and commits stay pending.
+ _log(f"kaggle auth failed ({type(exc).__name__}) and this pass requires it")
+ print(
+ "::error title=Kaggle authentication failed::the collector could not "
+ f"authenticate ({type(exc).__name__}). Nothing was collected or reaped; "
+ "dispatched kernels keep billing and their commits stay pending until "
+ "the token is repaired."
+ )
+ return finish(1)
+ # A skip, not a failure, for the same reason gate.py skips: on a fork
+ # pull request the secret is withheld and there is nothing to fix.
+ _log(f"kaggle auth failed ({type(exc).__name__}); nothing collected")
+ return finish()
+
+ owner = launch.username_of(api)
+ result["owner"] = owner
+ _log(f"authenticated as {owner}")
+
+ # The budget starts BEFORE the listing: five slow pages under the socket
+ # timeout are minutes, and a deadline started after them is that much later
+ # than the job timeout was sized for.
+ deadline = time.time() + BUDGET_SEC
+ ours = find_ours(api, max_age_hours = args.max_age_hours, deadline = deadline)
+ _log(f"{len(ours)} kernel(s) of ours on this account")
+
+ for entry in ours:
+ if time.time() >= deadline:
+ # Safe by construction: nothing was deleted that was not first
+ # collected. Logged so a partial pass is not read as an empty one.
+ _log("collection budget spent; the rest is left for the next pass")
+ break
+ result["kernels"].append(
+ collect_one(
+ api,
+ entry,
+ outdir,
+ args.expect,
+ args.max_age_hours,
+ delete = not args.no_delete,
+ deadline = deadline,
+ )
+ )
+
+ result["statuses"] = statuses_from(result["kernels"], args.target_url)
+
+ if args.sha:
+ want = args.sha.strip().lower()
+ # Prefix match either way: SLUG_SHA_LEN characters now, eight before,
+ # and both can be on the account at once.
+ result["in_flight_for_sha"] = any(
+ k.get("sha")
+ and (want.startswith(k["sha"]) or k["sha"].startswith(want))
+ and k.get("verdict") == "pending"
+ and (not args.kind or k.get("kind") == args.kind)
+ and k.get("slot", "1") == str(args.slot or "1").strip()
+ for k in result["kernels"]
+ )
+ _log(f"in flight for {want}: {result['in_flight_for_sha']}")
+
+ return finish()
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/kaggle_t4_ci/gate.py b/.github/scripts/kaggle_t4_ci/gate.py
index efe1033dedb..d1c11f6c1ef 100644
--- a/.github/scripts/kaggle_t4_ci/gate.py
+++ b/.github/scripts/kaggle_t4_ci/gate.py
@@ -358,22 +358,56 @@ def scaled_reserve(reserve_hours: float, total_hours: float, basis_hours: float)
return round(reserve_hours * (total_hours / basis_hours), 3)
-def weighted_pick(run_id: str, weights: dict[str, float]) -> tuple[str, float]:
- """Deterministic weighted choice of account, keyed on the run id ALONE.
+def in_flight_for_commit(
+ own_busy: list[str],
+ head_sha: str,
+ kind: str,
+ slot: str = "1",
+) -> str | None:
+ """The ref of a busy kernel of ours already running THIS commit, kind and
+ slot, or None.
+
+ The slot is part of the identity: a slot-1 kernel must not stand a slot-2
+ dispatch down, while a retry of the slot-2 run itself must be.
+
+ Asked of every account surveyed, not only the one picked: a handover lands
+ a retry on the other account, whose GPU job collects with only its own
+ token and would see nothing in flight.
+ """
+ sha = (head_sha or "").strip().lower()
+ slot = str(slot or "1").strip()
+ if not sha or not kind:
+ return None
+ import launch # noqa: PLC0415 (sibling script, loaded lazily so the gate imports alone)
+
+ for entry in own_busy:
+ ref = entry.split(" (", 1)[0]
+ parsed = launch.parse_slug(ref)
+ if not parsed or parsed.get("kind") != kind or not parsed.get("sha"):
+ continue
+ if parsed.get("slot", "1") != slot:
+ continue
+ if sha.startswith(parsed["sha"]) or parsed["sha"].startswith(sha):
+ return ref
+ return None
- Not the attempt, for the same reason `sampled_in` excludes it and a sharper
- one: a re-run of a run whose kernels are still in flight must return to the
- SAME account, or the second attempt pushes to an account the first one is
- not watching and the first one's kernels are reaped by nobody.
+
+def weighted_pick(key: str, weights: dict[str, float]) -> tuple[str, float]:
+ """Deterministic weighted choice of account, keyed on the COMMIT under test
+ (the run id when no commit is known).
+
+ The commit rather than the run or the attempt: every run of one commit
+ must land on the account already holding that commit's kernel, or the
+ collector on the other account sees nothing in flight, dispatches a
+ duplicate, and reaps nothing.
Salted differently from `sampled_in` so the two draws off one run id are
- independent -- without the salt, whether a run is sampled in would correlate
- with which account it lands on, and one account would quietly get a
- different SHARE of the forced runs than of the sampled ones.
+ independent; without the salt one account would get a different SHARE of
+ the forced runs than of the sampled ones.
"""
ids = sorted(weights)
total = sum(max(0.0, weights[i]) for i in ids)
- digest = hashlib.sha256(("account:" + run_id).encode("utf-8")).hexdigest()
+ digest = hashlib.sha256(("account:" + key).encode("utf-8")).hexdigest()
draw = (int(digest[:8], 16) % 1_000_000) / 1_000_000.0
if not ids or total <= 0:
return (ids[0] if ids else ""), draw
@@ -664,6 +698,25 @@ def main() -> int:
)
ap.add_argument("--run-id", default = os.environ.get("GITHUB_RUN_ID", "0"))
ap.add_argument("--run-attempt", default = os.environ.get("GITHUB_RUN_ATTEMPT", "1"))
+ ap.add_argument(
+ "--head-sha",
+ default = "",
+ help = "the commit under test; the account draw is keyed on it so every run of one "
+ "commit lands on the account holding its kernel",
+ )
+ ap.add_argument(
+ "--kind",
+ default = "",
+ help = "notebook or studio: with --head-sha, a kernel of this kind already running "
+ "this commit on ANY account stands the run down instead of dispatching a duplicate",
+ )
+ ap.add_argument(
+ "--slot",
+ default = "1",
+ help = "the workflow's session slot input. Slot 2 is an opt-in SECOND session on "
+ "the same commit beside slot 1: only a kernel already running this commit in "
+ "the SAME slot stands the run down",
+ )
ap.add_argument("--force", default = "false", help = "workflow_dispatch force input")
ap.add_argument("--labels", default = "", help = "comma or newline separated PR labels")
ap.add_argument("--label-name", default = "kaggle-t4-ci")
@@ -857,12 +910,11 @@ def main() -> int:
print("[gate] accounts " + json.dumps(list(probes.values())), flush = True)
_out("accounts", json.dumps(list(probes.values())))
- # An account whose quota could not be read has no weight, because a weight
- # is what its plan says and we did not get to hear it. It stays a CANDIDATE
- # -- the order below still reaches it -- so an unreadable answer costs the
- # account its share of the traffic and not its place in the queue.
+ # An unreadable quota means no weight but still a CANDIDATE: it costs the
+ # account its share of the traffic, not its place in the queue.
weights = {i: p["total_hours"] for i, p in probes.items() if p.get("total_hours")}
- sampled_account, account_draw = weighted_pick(args.run_id, weights)
+ account_key = (args.head_sha or "").strip().lower() or str(args.run_id)
+ sampled_account, account_draw = weighted_pick(account_key, weights)
if weights:
share = max(0.0, weights.get(sampled_account, 0.0)) / sum(weights.values())
print(
@@ -878,11 +930,45 @@ def main() -> int:
flush = True,
)
- # The sampled account first, then the rest in declaration order. Only the
- # account actually being considered pays for a survey, which is the
- # expensive call here.
+ # Sampled account first, then declaration order. Only an account actually
+ # considered pays for a survey, the expensive call here.
order = [sampled_account] + [i for i in probes if i != sampled_account]
handovers: list[str] = []
+ surveys: dict[str, dict] = {}
+ # ONE survey budget for the whole gate: two bounded surveys back to back
+ # outlive the job timeout, and a runner killed mid-question is a red rather
+ # than the soft stand-down an incomplete survey gives.
+ survey_deadline = time.monotonic() + SURVEY_BUDGET_SEC
+
+ def _survey(account_id: str) -> dict:
+ if account_id not in surveys:
+ surveys[account_id] = survey_kernels(
+ clients[account_id],
+ budget_sec = max(0.0, survey_deadline - time.monotonic()),
+ )
+ return surveys[account_id]
+
+ # Ask EVERY usable account about this commit before choosing one: an
+ # earlier run may have handed the commit to the other account, and a retry
+ # finding the preferred one free again would never look at the kernel still
+ # running. Capacity for a NEW launch is beside the point here.
+ slot = str(args.slot or "1").strip()
+ if args.head_sha and args.kind:
+ for account_id in order:
+ if not account_id or clients.get(account_id) is None:
+ continue
+ try:
+ survey = _survey(account_id)
+ except Exception: # noqa: BLE001
+ continue # reported below, where the account is considered
+ already = in_flight_for_commit(survey["own"], args.head_sha, args.kind, slot)
+ if already:
+ return _decide(
+ False,
+ f"a {args.kind} kernel for this commit (slot {slot}) is already running "
+ f"on account {account_id} ({already}); its result arrives as the commit "
+ "status, so nothing is dispatched",
+ )
for account_id in order:
if not account_id:
@@ -892,9 +978,8 @@ def main() -> int:
handovers.append(f"account {account_id} {record['outcome']}")
continue
- api = clients[account_id]
try:
- survey = survey_kernels(api)
+ survey = _survey(account_id)
except Exception as exc: # noqa: BLE001
record["outcome"] = "capacity_unreadable"
record["error"] = type(exc).__name__
diff --git a/.github/scripts/kaggle_t4_ci/launch.py b/.github/scripts/kaggle_t4_ci/launch.py
index 9ce3eac7e6e..7f79a29270d 100644
--- a/.github/scripts/kaggle_t4_ci/launch.py
+++ b/.github/scripts/kaggle_t4_ci/launch.py
@@ -86,6 +86,88 @@ TERMINAL_BAD = {"ERROR", "CANCEL_REQUESTED", "CANCEL_ACKNOWLEDGED"}
_STATUS_RE = re.compile(r"KernelWorkerStatus\.(?P<status>[A-Z_]+)")
+# The slug is the ONLY record that survives the runner.
+#
+# Dispatch mode exits minutes after the push and a DIFFERENT job collects,
+# possibly days later, with no artifact or database between them. Kaggle's
+# kernel listing is that database, and a kernel carries exactly one field we
+# control: its name.
+#
+# unsloth-t4-ci-n1a2b3c4d5e6f-9f8e
+# ^^^^^^^^^^^^^^^^^^
+# ||||||||||||| uniqueness, so two runs on one commit do not
+# |||||||||||| collide (a re-run, or notebook slot 1 and 2)
+# |kind which workflow to report against
+# sha12 which commit the result belongs to. Twelve,
+# not eight: two commits sharing eight hex
+# characters give indistinguishable kernels, and
+# the commits API then 422s the prefix on every
+# pass, so the status stays pending forever.
+#
+# `OWN_KERNEL_PREFIX` in gate.py is the first 14 characters and MUST keep
+# matching: it is how the survey tells our kernels from a human's, and one it
+# cannot recognise is not counted against Kaggle's 2-session cap.
+SLUG_PREFIX = "unsloth-t4-ci-"
+
+# One character each, so the slug survives `_slugify` and stays under Kaggle's
+# length limit. A wire format, not an enum: a collector reads it back days later.
+KIND_CODES = {"notebook": "n", "studio": "s"}
+CODE_KINDS = {v: k for k, v in KIND_CODES.items()}
+
+# Legacy slugs (`unsloth-t4-ci-<8 hex>`) are still OURS and must still be
+# reaped, but carry no commit to report against. The alternation reads 8 to 40
+# hex rather than making the collector guess from length.
+SLUG_SHA_LEN = 12
+# The `slot` input, when it is not 1, sits between the dash and the uid:
+# `-2<uid4>`. Slot 2 is a deliberate SECOND session on the same commit, so the
+# duplicate check has to say "this commit, this kind, THIS slot". Absent means
+# slot 1, so every slug pushed before the slot existed still reads.
+CI_SLUG_RE = re.compile(
+ r"^(?:(?P<owner>[^/]+)/)?"
+ + re.escape(SLUG_PREFIX)
+ + r"(?:(?P<kind>[a-z])(?P<sha>[0-9a-f]{8,40})-(?P<slot>[2-9])?(?P<uid>[0-9a-f]{4})"
+ r"|(?P<legacy>[0-9a-f]{8}))$"
+)
+
+
+def parse_slug(slug: str) -> dict | None:
+ """What a kernel name says about itself, or None if it is not ours.
+
+ None for anything unrecognised is the collector's safety property: it
+ enumerates a shared ACCOUNT and deletes what it collects, so a collector
+ that guessed would eventually delete someone's work.
+ """
+ match = CI_SLUG_RE.match(slug or "")
+ if match is None:
+ return None
+ return {
+ "owner": match.group("owner"),
+ "kind": CODE_KINDS.get(match.group("kind") or "", None),
+ "sha": match.group("sha"),
+ "slot": match.group("slot") or "1",
+ "legacy": match.group("legacy") is not None,
+ }
+
+
+def slug_name(
+ kind: str = "",
+ commit_sha: str = "",
+ slot: str = "1",
+) -> str:
+ """The name for a kernel about to be pushed.
+
+ Falls back to the legacy random form when no commit is named, as a local
+ probe or an older workflow does. A dispatched run always names one.
+ """
+ code = KIND_CODES.get(kind, "")
+ sha = (commit_sha or "").strip().lower()[:SLUG_SHA_LEN]
+ slot = str(slot or "1").strip()
+ mark = slot if len(slot) == 1 and slot in "23456789" else ""
+ if code and len(sha) == SLUG_SHA_LEN and all(c in "0123456789abcdef" for c in sha):
+ return f"{SLUG_PREFIX}{code}{sha}-{mark}{uuid.uuid4().hex[:4]}"
+ return f"{SLUG_PREFIX}{uuid.uuid4().hex[:8]}"
+
+
PUSH_ATTEMPTS = 4
PUSH_BACKOFF_SEC = 45
@@ -181,7 +263,11 @@ OUTPUT_PAGE_LIMIT = 20
READ_CHUNK_BYTES = 1 << 16
-def worst_case_seconds(max_wait: int, kernels: int) -> int:
+def worst_case_seconds(
+ max_wait: int,
+ kernels: int,
+ dispatch: bool = False,
+) -> int:
"""Wall clock ONE invocation of this launcher can take, from its constants.
Every phase that can keep a pushed kernel UP is in it, because a kernel
@@ -209,6 +295,11 @@ def worst_case_seconds(max_wait: int, kernels: int) -> int:
+ sum(PUSH_BACKOFF_SEC * 2**i for i in range(PUSH_ATTEMPTS - 1))
+ (PUSH_ATTEMPTS - 1) * one_delete
)
+ if dispatch:
+ # Dispatch stops after the pushes, so the phases below are unreachable
+ # and counting them would make the pre-push guard demand ~2h for a job
+ # that exits in five. The retry _discard()s stay in: those run in push().
+ return kernels * per_push
return (
max(kernels * per_push, max_wait)
+ EVIDENCE_BUDGET_SEC
@@ -499,6 +590,9 @@ def push(
kernel_timeout_sec: int,
accelerator: str = "NvidiaTeslaT4",
attempted: list[str] | None = None,
+ kind: str = "",
+ commit_sha: str = "",
+ slot: str = "1",
) -> dict:
"""Push as a fresh private kernel. Every attempt gets its own slug.
@@ -527,7 +621,6 @@ def push(
``MemoryError``) would otherwise take the slug of a push Kaggle may have
ACCEPTED with it, leaving a kernel nothing will delete. See main().
"""
- base = _slugify("unsloth t4 ci")[:32]
attempted = [] if attempted is None else attempted
def _discard(slug: str) -> None:
@@ -545,22 +638,23 @@ def push(
for attempt in range(PUSH_ATTEMPTS):
if attempted:
_discard(attempted[-1])
- slug_name = f"{base}-{uuid.uuid4().hex[:8]}"
+ name = slug_name(kind, commit_sha, slot)
# The slug derives from the TITLE, not the metadata id: a mismatch
- # files the kernel at an unexpected address and every later
- # status/output call 403s, so assert the round trip.
- title = slug_name.replace("-", " ")
- assert _slugify(title) == slug_name, f"title {title!r} slugifies to {_slugify(title)!r}"
- attempted.append(f"{user}/{slug_name}")
+ # files the kernel at an unexpected address, every later
+ # status/output call 403s and no collector can attribute it. So
+ # assert the round trip.
+ title = name.replace("-", " ")
+ assert _slugify(title) == name, f"title {title!r} slugifies to {_slugify(title)!r}"
+ attempted.append(f"{user}/{name}")
for stale in workdir.glob("*.ipynb"):
stale.unlink()
- code_file = workdir / f"{slug_name}.ipynb"
+ code_file = workdir / f"{name}.ipynb"
shutil.copy(notebook, code_file)
(workdir / "kernel-metadata.json").write_text(
json.dumps(
{
- "id": f"{user}/{slug_name}",
+ "id": f"{user}/{name}",
"title": title,
"code_file": code_file.name,
"language": "python",
@@ -650,9 +744,13 @@ def _already_gone(text: str) -> bool:
return any(marker in text.lower() for marker in GONE_MARKERS)
-def delete_kernel(slug: str) -> bool:
+def delete_kernel(slug: str, deadline: float | None = None) -> bool:
"""Delete one kernel, and answer whether Kaggle actually deleted it.
+ ``deadline`` (absolute ``time.time()``) bounds the whole call: attempts are
+ clamped to what is left and none starts past it, so a release phase sharing
+ one deadline ends when it says it will rather than three attempts after.
+
``subprocess.run`` does not raise on a nonzero exit, so the caller used to
record every slug as released whatever came back: a refused delete, an
expired token, or the case that made this visible, where the pinned client
@@ -686,12 +784,21 @@ def delete_kernel(slug: str) -> bool:
this refuses needs a human, which is what the caller's warning is for.
"""
for attempt in range(DELETE_ATTEMPTS):
+ timeout = DELETE_SUBPROCESS_TIMEOUT_SEC
+ if deadline is not None:
+ left = deadline - time.time()
+ if left <= 0:
+ _log(
+ f"delete {slug}: budget spent before attempt {attempt + 1}; left for the next pass"
+ )
+ return False
+ timeout = min(timeout, left)
try:
proc = subprocess.run(
["kaggle", "kernels", "delete", slug, "-y"],
capture_output = True,
text = True,
- timeout = DELETE_SUBPROCESS_TIMEOUT_SEC,
+ timeout = timeout,
)
except Exception as exc: # noqa: BLE001
_log(f"delete {slug} did not run: {type(exc).__name__}")
@@ -941,6 +1048,10 @@ def fetch_evidence(
continue
url = entry.get("url") or entry.get("urlNullable")
if not url:
+ # Listed but not downloadable is missing evidence: judged without
+ # it, a lost failing payload reads as whatever survived.
+ _log(f"{name} was listed without a download URL; evidence incomplete")
+ truncated = True
continue
call_timeout = _time_left(deadline, timeout)
if call_timeout is None:
@@ -1019,6 +1130,10 @@ def extract_reports(outdir: Path) -> list[dict]:
parsed = json.loads(blob)
except json.JSONDecodeError:
continue
+ # A report is an object; `.get` on anything else that parses raises
+ # out of the collector and wedges every later pass on this evidence.
+ if not isinstance(parsed, dict):
+ continue
key = f"{parsed.get('label')}|{parsed.get('model')}"
if key in seen:
continue
@@ -1032,11 +1147,23 @@ def extract_reports(outdir: Path) -> list[dict]:
nb = json.loads(nb_path.read_text(encoding = "utf-8", errors = "replace"))
except Exception: # noqa: BLE001
continue
- for cell in nb.get("cells", []):
- for output in cell.get("outputs", []):
+ # Valid JSON is not necessarily a notebook, and one malformed file must
+ # not take its neighbours' reports with it: an exception here used to
+ # reach the collector, which then judged the kernel infra, posted
+ # success and released it, hiding a real failure behind a mangled file.
+ if not isinstance(nb, dict):
+ continue
+ for cell in nb.get("cells") or []:
+ if not isinstance(cell, dict):
+ continue
+ for output in cell.get("outputs") or []:
+ if not isinstance(output, dict):
+ continue
text = output.get("text") or ""
if isinstance(text, list):
- text = "".join(text)
+ text = "".join(str(t) for t in text)
+ if not isinstance(text, str):
+ continue
_consume(text)
for log_path in sorted(outdir.rglob("kernel.log")):
raw = log_path.read_text(encoding = "utf-8", errors = "replace")
@@ -1152,6 +1279,33 @@ def main() -> int:
ap.add_argument(
"--keep-kernel", action = "store_true", help = "do not delete the kernel after collecting"
)
+ ap.add_argument(
+ "--dispatch",
+ action = "store_true",
+ help = "push and EXIT, without waiting, collecting or deleting. The kernel "
+ "is collected later by collect.py, which finds it by its slug",
+ )
+ ap.add_argument(
+ "--commit-sha",
+ default = "",
+ help = "the commit under test. Written into the slug, which is the only "
+ "record a later collector has of what the result is about",
+ )
+ ap.add_argument(
+ "--kind",
+ default = "",
+ choices = ("", *KIND_CODES),
+ help = "which workflow is dispatching, so the collector knows which "
+ "commit status context to report the result under",
+ )
+ ap.add_argument(
+ "--slot",
+ default = "1",
+ choices = tuple("123456789"),
+ help = "the workflow's session slot. Written into the slug when it is not 1, "
+ "so a retry of a slot-2 run is recognised as the same session and a slot-2 "
+ "dispatch beside slot 1 is not",
+ )
ap.add_argument(
"--deadline-epoch",
type = int,
@@ -1161,6 +1315,29 @@ def main() -> int:
)
args = ap.parse_args()
+ # A dispatched kernel is left running, which is exactly what --keep-kernel
+ # already means down to the registry bookkeeping. Reusing the flag rather
+ # than giving release() a second way not to delete: two conditions guarding
+ # one deletion is how one ends up wrong, and this one bills GPU quota.
+ if args.dispatch:
+ args.keep_kernel = True
+ # Refused rather than defaulted: a slug with no commit runs, costs quota
+ # and reports to nobody. A usage error at the only moment it is cheap.
+ if not args.commit_sha or not args.kind:
+ ap.error(
+ "--dispatch requires --commit-sha and --kind: without both, the "
+ "kernel runs and no collector can attribute its result"
+ )
+ # A commit, not a ref: `slug_name` carries only hex and falls back to
+ # the legacy unattributable form, so a branch or tag here bills and
+ # reports to nobody. The workflows resolve refs; this checks they did.
+ if not re.fullmatch(r"[0-9a-fA-F]{12,40}", args.commit_sha.strip()):
+ ap.error(
+ f"--commit-sha must be a hex commit id (12 to 40 characters), got "
+ f"{args.commit_sha!r}: a ref cannot be written into the slug, so "
+ "the kernel's result could never be attributed to a commit"
+ )
+
# Before the first network call, and globally: the Kaggle client has no
# per-call timeout of its own. See SOCKET_TIMEOUT_SEC.
socket.setdefaulttimeout(SOCKET_TIMEOUT_SEC)
@@ -1248,7 +1425,7 @@ def main() -> int:
"""
if not args.deadline_epoch:
return True
- need = worst_case_seconds(args.max_wait, len(args.notebook))
+ need = worst_case_seconds(args.max_wait, len(args.notebook), dispatch = args.dispatch)
left = int(args.deadline_epoch - time.time())
if left >= need:
_log(f"{left}s left of the job deadline, worst case {need}s ({after})")
@@ -1398,6 +1575,9 @@ def main() -> int:
args.user,
args.kernel_timeout_sec,
attempted = entry["attempted"],
+ kind = args.kind,
+ commit_sha = args.commit_sha,
+ slot = args.slot,
)
entry["slug"] = pushed.get("slug")
entry["push_error"] = (
@@ -1416,6 +1596,22 @@ def main() -> int:
# single-kernel shape.
result["slug"] = live[0]["slug"]
+ # DISPATCH MODE ENDS HERE. `dispatched` is deliberately NOT `pass`: all
+ # this job proved is that Kaggle accepted a push. The real verdict comes
+ # later from collect.py as a commit status on the sha in the slug.
+ if args.dispatch:
+ result["verdict"] = "dispatched"
+ result["dispatched"] = [{"slug": k["slug"], "notebook": k["notebook"]} for k in live]
+ result["commit_sha"] = args.commit_sha
+ result["kind"] = args.kind
+ result["reason"] = (
+ f"{len(live)} kernel(s) pushed and left running: "
+ + ", ".join(k["slug"] for k in live)
+ + f". The result for {args.commit_sha[:8]} will be posted as a commit "
+ "status by the collector, not by this job"
+ )
+ return finish()
+
# The deadline above is shared, not one per kernel: they run
# concurrently, so consuming the ceiling per kernel would let a
# two-kernel run wait twice its own stated bound.
diff --git a/.github/scripts/kaggle_t4_ci/post_statuses.py b/.github/scripts/kaggle_t4_ci/post_statuses.py
new file mode 100644
index 00000000000..7fc2bcd5690
--- /dev/null
+++ b/.github/scripts/kaggle_t4_ci/post_statuses.py
@@ -0,0 +1,191 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Post the commit statuses collect.py decided on, and say which ones landed.
+
+The only step holding a GitHub credential, and it holds no Kaggle one:
+collect.py judges, this posts, neither can leak the other's token. A script
+rather than a shell loop in three workflows so the record stays JSON end to end
+and each value reaches `gh` as an argument, never through a shell.
+
+THE SHA HAS TO BE EXPANDED FIRST. The slug carries an abbreviation (a full sha
+does not fit Kaggle's slug limit) and the statuses API refuses one, measured
+against a real repository:
+
+ POST /statuses/2ecb19df
+ 422 "Sha must be a valid hex object ID"
+
+A sha that cannot be resolved (force-pushed away) is recorded as `unresolved`
+so the kernel is released rather than retried forever.
+
+``posted.json`` feeds collect.py --delete-collected: a kernel whose status did
+not post is KEPT on Kaggle for the next pass. That ordering (post, then delete)
+is why this is a separate step.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from collect import STATUS_CONTEXTS # noqa: E402
+
+STATES = ("error", "failure", "pending", "success")
+FULL_SHA = re.compile(r"^[0-9a-f]{40}$")
+
+
+def _gh(args: list[str]) -> tuple[int, str, str]:
+ proc = subprocess.run(["gh", *args], capture_output = True, text = True)
+ return proc.returncode, (proc.stdout or "").strip(), (proc.stderr or "").strip()
+
+
+# What GitHub says when the commit is genuinely not there, measured:
+#
+# gh api repos/unslothai/unsloth/commits/deadbeef00
+# {"message":"No commit found for SHA: deadbeef00", ..., "status":"422"}
+#
+# Only that message releases the kernel. Status codes do not say it: 404 is
+# also an unreadable repository, 422 an ambiguous abbreviation, and reading
+# either as "gone" would delete the only copy of the result.
+MISSING_MARKERS = ("no commit found",)
+
+
+def resolve_sha(repo: str, sha: str) -> tuple[str, str | None]:
+ """``("ok", full_sha)``, ``("missing", None)`` when the repository no
+ longer has the commit, or ``("error", None)`` when the lookup itself
+ failed and nothing is known about the commit."""
+ code, out, err = _gh(["api", f"repos/{repo}/commits/{sha}", "-q", ".sha"])
+ if code == 0 and FULL_SHA.match(out):
+ return "ok", out
+ text = f"{out} {err}".lower()
+ if any(m in text for m in MISSING_MARKERS):
+ return "missing", None
+ return "error", None
+
+
+def post_one(repo: str, full_sha: str, status: dict) -> bool:
+ code, _out, _err = _gh(
+ [
+ "api",
+ f"repos/{repo}/statuses/{full_sha}",
+ "-f",
+ f"state={status['state']}",
+ "-f",
+ f"context={status['context']}",
+ "-f",
+ f"description={status['description']}",
+ "-f",
+ f"target_url={status.get('target_url') or ''}",
+ "--silent",
+ ]
+ )
+ return code == 0
+
+
+def valid(status: dict) -> str | None:
+ """Why this record must not be posted, or None if it is well formed.
+
+ Checked here because this is the process holding the token: a state outside
+ the API's four, or a context not ours, is a record we must not sign.
+ """
+ if status.get("state") not in STATES:
+ return f"state {status.get('state')!r} is not one of {STATES}"
+ if status.get("context") not in STATUS_CONTEXTS.values():
+ return f"context {status.get('context')!r} is not a Kaggle CI context"
+ if not re.fullmatch(r"[0-9a-f]{8,40}", str(status.get("sha") or "")):
+ return f"sha {status.get('sha')!r} is not a hex commit id"
+ return None
+
+
+def merge_by_commit(resolved: list[tuple[str, dict]]) -> list[tuple[str, dict]]:
+ """One status per (full sha, context), failure winning, every slug named.
+
+ Merged HERE and not in the collector, because only a resolved sha says
+ whether two slugs name one commit: an 8 and a 12 character slug for one
+ commit resolve alike, two commits sharing 8 characters do not.
+ """
+ out: dict[tuple[str, str], dict] = {}
+ for full, status in resolved:
+ key = (full, status["context"])
+ prior = out.get(key)
+ if prior is None:
+ out[key] = dict(status, slugs = list(status.get("slugs") or [status.get("slug")]))
+ continue
+ slugs = prior["slugs"] + list(status.get("slugs") or [status.get("slug")])
+ if status["state"] == "failure" and prior["state"] != "failure":
+ out[key] = dict(status, slugs = slugs)
+ else:
+ prior["slugs"] = slugs
+ return [(k[0], s) for k, s in out.items()]
+
+
+def post_all(statuses: list[dict], repo: str) -> dict:
+ outcome: dict[str, list[str]] = {"ok": [], "failed": [], "unresolved": [], "invalid": []}
+ resolved: list[tuple[str, dict]] = []
+ for status in statuses:
+ slugs = list(status.get("slugs") or [status.get("slug")])
+ why = valid(status)
+ if why:
+ print(f"::warning title=Malformed status record::{why}; not posted")
+ outcome["invalid"].extend(slugs)
+ continue
+ found, full = resolve_sha(repo, status["sha"])
+ if found == "missing":
+ print(
+ f"::warning title=Could not resolve a collected commit::{status['sha']} is "
+ f"no longer a commit in this repository, so its {status['context']} result "
+ "cannot be posted. The kernel is released; nothing will ever post for it."
+ )
+ outcome["unresolved"].extend(slugs)
+ continue
+ if found != "ok":
+ print(
+ f"::warning title=Commit lookup failed::could not ask GitHub about "
+ f"{status['sha']} this pass; the kernel is kept so the next pass can try again"
+ )
+ outcome["failed"].extend(slugs)
+ continue
+ resolved.append((full, status))
+ for full, status in merge_by_commit(resolved):
+ slugs = status["slugs"]
+ print(f"posting {status['context']}={status['state']} for {full}")
+ if post_one(repo, full, status):
+ outcome["ok"].extend(slugs)
+ else:
+ print(
+ f"::warning title=Could not post a commit status::{status['context']} for "
+ f"{full}; the kernel is kept so the next pass can try again"
+ )
+ outcome["failed"].extend(slugs)
+ return outcome
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--result", required = True, help = "collect_result.json from collect.py")
+ ap.add_argument("--out", required = True, help = "where to write posted.json")
+ ap.add_argument("--repo", default = os.environ.get("GITHUB_REPOSITORY", ""))
+ args = ap.parse_args()
+ if not args.repo:
+ ap.error("--repo (or GITHUB_REPOSITORY) is required")
+
+ data = json.loads(Path(args.result).read_text(encoding = "utf-8"))
+ statuses = data.get("statuses") or []
+ outcome = post_all(statuses, args.repo)
+ Path(args.out).write_text(json.dumps(outcome, indent = 2), encoding = "utf-8")
+ if not statuses:
+ print("no statuses to post this pass")
+ # Red when a verdict could not be delivered: the kernel is kept for the
+ # next pass, but a delivery failure must not look like a quiet account.
+ return 1 if outcome["failed"] else 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/workflows/kaggle-collect.yml b/.github/workflows/kaggle-collect.yml
new file mode 100644
index 00000000000..a9c491d6d11
--- /dev/null
+++ b/.github/workflows/kaggle-collect.yml
@@ -0,0 +1,214 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+#
+# Collect the Kaggle kernels the GPU workflows dispatched and left running.
+#
+# WHY THIS EXISTS
+#
+# The GPU workflows used to hold a runner for the whole Kaggle session: 41.5
+# and 18.7 minutes, measured on runs 33479481067 and 33486360729, almost all of
+# it polling, against a queue measured 400-670 deep.
+#
+# So those jobs now PUSH AND EXIT (`launch.py --dispatch`), and two independent
+# routes collect: the next GPU run on the same account collects before it
+# dispatches, and THIS workflow covers a quiet repository on a schedule. The
+# schedule is not a nicety; a kernel here has been measured ignoring Kaggle's
+# own `-t` timeout for over two hours, billing the whole time.
+#
+# WHAT IT PUBLISHES
+#
+# A commit status per collected kernel, `kaggle-t4-notebook` or
+# `kaggle-studio-gpu`, against the sha in the kernel's slug. THAT is the check
+# meaning "the GPU tests passed"; the dispatching job proves only that Kaggle
+# accepted a push. Neither context can be a REQUIRED check while the GPU
+# workflows sample and path-filter: a commit the gate skips gets no status, and
+# a required context that never arrives blocks the merge.
+#
+# ACCOUNTS
+#
+# A static two-element matrix, one job per account, unlike the gate's
+# quota-weighted draw: the gate chooses where to SPEND a session, this spends
+# none and must look at both every pass. One token per job, never both in one
+# step, as in the GPU workflows.
+name: Kaggle collect
+
+on:
+ schedule:
+ # Every 10 minutes, offset off the hour: a few passes per 20-40 minute
+ # kernel, without the workflow becoming a queue item itself.
+ - cron: '3,13,23,33,43,53 * * * *'
+ workflow_dispatch:
+ inputs:
+ max_age_hours:
+ description: 'Delete a still-running kernel older than this many hours'
+ type: string
+ default: '3'
+ no_delete:
+ description: 'Collect and report without deleting (inspection only; a kernel left up keeps billing)'
+ type: boolean
+ default: false
+
+permissions:
+ contents: read
+
+# Never two scheduled passes at once. This does NOT serialise against the GPU
+# jobs, which collect under their own groups, so two collectors CAN see one
+# terminal kernel. Safe by construction in collect.py rather than by a lock: a
+# vanished kernel is `gone` and posts nothing, an incomplete download keeps the
+# kernel, and a status comes only from complete evidence, so the worst case is
+# the same verdict twice. Not cancel-in-progress: a pass midway through a
+# download must finish, or the next starts from nothing.
+concurrency:
+ group: kaggle-collect
+ cancel-in-progress: false
+
+jobs:
+ collect:
+ name: Collect
+ runs-on: ubuntu-latest
+ # collect.py's BUDGET_SEC (900s) plus RELEASE_BUDGET_SEC (600s) plus setup
+ # and posting, so the script's budgets stop a long pass rather than the job
+ # being killed mid-download.
+ timeout-minutes: 30
+ permissions:
+ contents: read
+ # The point of the workflow: without it the verdict reaches nobody.
+ statuses: write
+ strategy:
+ # One unreachable account must not stop the other, and either failing is
+ # still a failure.
+ fail-fast: false
+ matrix:
+ include:
+ - account_id: '1'
+ secret_name: KAGGLE_API_TOKEN
+ - account_id: '2'
+ secret_name: KAGGLE_API_TOKEN_2
+ steps:
+ - name: Harden runner (audit)
+ uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a
+ with:
+ egress-policy: audit
+
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # The collector holds the Kaggle token; only the posting step gets
+ # GH_TOKEN, and checkout persists the job token into .git by default.
+ persist-credentials: false
+
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+ with:
+ python-version: '3.12'
+
+ - name: Install the Kaggle client
+ # As in the GPU workflows: 1.7.4.5's authenticate() cannot read
+ # KAGGLE_API_TOKEN, the only credential this job has, so every pass on
+ # it collected nothing.
+ run: python -m pip install --quiet 'kaggle==2.2.4'
+
+ - name: Collect this account's kernels
+ id: collect
+ env:
+ # Exactly one account, indexed by name. A ternary would hand over the
+ # other account's token whenever this secret is empty, and the
+ # collector would delete kernels it is not reporting for.
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
+ # --require-auth: this job holds the repository's own secrets, so an
+ # auth failure is a broken token and must be red, not the quiet green a
+ # fork gets. An EMPTY token is different: the matrix is static, so a
+ # repository with one Kaggle account leaves the second secret unset and
+ # that leg warns and passes. --no-delete always; the last step releases
+ # the kernels once their statuses posted. The report count comes from
+ # each kernel's own evidence, so a five-payload kernel that lost four is
+ # judged against five rather than a flat guess.
+ run: |
+ python .github/scripts/kaggle_t4_ci/collect.py \
+ --outdir kaggle_collected \
+ --require-auth \
+ --no-delete \
+ --max-age-hours '${{ inputs.max_age_hours || '3' }}' \
+ --target-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+
+ # Post FIRST, delete SECOND: a verdict that does not reach GitHub leaves
+ # its kernel up for the next pass, and deleting first would make one API
+ # failure the permanent loss of a verdict. A script rather than a shell
+ # loop so each field reaches `gh` as its own argument.
+ - name: Post the commit statuses
+ id: post
+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/post_statuses.py \
+ --result kaggle_collected/collect_result.json \
+ --out kaggle_collected/posted.json
+
+ - name: Release the kernels whose verdicts were delivered
+ if: always() && !inputs.no_delete && hashFiles('kaggle_collected/collect_result.json') != ''
+ env:
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/collect.py \
+ --outdir kaggle_collected \
+ --delete-collected kaggle_collected/collect_result.json \
+ --posted kaggle_collected/posted.json
+
+ # The Studio payload smuggles its screenshots and logs home in its own
+ # cell output, and this pass collects most Studio kernels, so it has to
+ # put them back. One kernel per call: the bundle is reassembled from
+ # numbered chunks and one walk over two would splice them together.
+ - name: Unpack the Playwright evidence
+ if: always() && hashFiles('kaggle_collected/**/*_output.ipynb') != ''
+ continue-on-error: true
+ run: |
+ for dir in kaggle_collected/*/; do
+ [ -d "$dir" ] || continue
+ python .github/scripts/kaggle_studio_ci/collect_evidence.py \
+ --evidence "$dir" \
+ --outdir "studio_evidence/$(basename "$dir")"
+ done
+
+ # The posted verdict is one line; the notebooks and screenshots are what a
+ # red one is read from. continue-on-error because the statuses are already
+ # delivered and an artifact outage is not a collection failure.
+ - name: Upload evidence
+ if: always() && hashFiles('kaggle_collected/**') != ''
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: kaggle-collect-evidence-account-${{ matrix.account_id }}
+ path: |
+ kaggle_collected/**
+ studio_evidence/**
+ retention-days: 14
+ if-no-files-found: ignore
+
+ - name: Summarise
+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''
+ run: |
+ python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+ import json, pathlib
+ data = json.loads(pathlib.Path("kaggle_collected/collect_result.json").read_text())
+ # Collection runs --no-delete, so `deleted` there is always false; the
+ # release step wrote the real outcome beside it.
+ release = pathlib.Path("kaggle_collected/delete_result.json")
+ released = set(json.loads(release.read_text()).get("deleted") or []) if release.exists() else set()
+ print(f"### Kaggle collect: account {data.get('owner') or '?'}\n")
+ rows = data.get("kernels") or []
+ for k in rows:
+ k["deleted"] = bool(k.get("deleted")) or k.get("slug") in released
+ if not rows:
+ print("Nothing of ours was in flight on this account.\n")
+ else:
+ print("| kernel | commit | kind | state | verdict | deleted |")
+ print("| --- | --- | --- | --- | --- | --- |")
+ for k in rows:
+ print("| `{}` | `{}` | {} | {} | **{}** | {} |".format(
+ k["slug"].rsplit("/", 1)[-1], k.get("sha") or "-",
+ k.get("kind") or "-", k.get("state") or "-",
+ k.get("verdict") or "-", "yes" if k.get("deleted") else "no"))
+ print()
+ for k in rows:
+ if k.get("reason"):
+ print(f"- `{k['slug'].rsplit('/', 1)[-1]}`: {k['reason']}")
+ PY
diff --git a/.github/workflows/kaggle-t4-notebook-ci.yml b/.github/workflows/kaggle-t4-notebook-ci.yml
index 409021d13c8..a8cfd37de04 100644
--- a/.github/workflows/kaggle-t4-notebook-ci.yml
+++ b/.github/workflows/kaggle-t4-notebook-ci.yml
@@ -373,7 +373,10 @@ jobs:
# and exits 0); this simply does not start.
if: github.event.pull_request.head.repo.fork != true
runs-on: ubuntu-latest
- timeout-minutes: 10
+ # 15, the worst case with two accounts: each one's authentication, username
+ # and quota read at the socket ceiling, one shared survey budget, and the
+ # call still in flight when it expires. The suite asserts the arithmetic.
+ timeout-minutes: 15
outputs:
should_run: ${{ steps.decide.outputs.should_run }}
reason: ${{ steps.decide.outputs.reason }}
@@ -383,6 +386,9 @@ jobs:
# NAME, and the GPU job indexes the secrets context with it.
account: ${{ steps.decide.outputs.account }}
account_user: ${{ steps.decide.outputs.account_user }}
+ # The commit under test, resolved once: the GPU job tests THIS and never
+ # re-resolves a moving ref.
+ head_sha: ${{ steps.ref.outputs.head_sha }}
matrix: ${{ steps.decide.outputs.matrix }}
steps:
- name: Harden runner (audit)
@@ -464,6 +470,43 @@ jobs:
# than being interpolated into the shell, for the same reason the
# stale-approval warning below does: they are free text set by anyone with
# write access.
+ # THE COMMIT THE GPU JOB WILL TEST, which is not always this workflow's
+ # own: a dispatch naming `unsloth_ref` tests that ref. Keyed on github.sha
+ # instead, the in-flight check would look for the wrong kernel, and once
+ # the default branch moved two runs of one requested ref could be drawn
+ # onto different accounts and never see each other. A ref that does not
+ # resolve leaves the key EMPTY, so the draw falls back to the run id and
+ # the GPU job stands down on the same ref moments later.
+ - name: Name the commit under test
+ id: ref
+ env:
+ UNSLOTH_REF: ${{ inputs.unsloth_ref }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ run: |
+ if [ -z "$UNSLOTH_REF" ]; then
+ echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ if printf '%s' "$UNSLOTH_REF" | grep -Eq '^[0-9a-f]{40}$'; then
+ RESOLVED="$UNSLOTH_REF"
+ else
+ RESOLVED=''
+ for attempt in 1 2 3; do
+ LS="$(git ls-remote https://github.com/unslothai/unsloth "$UNSLOTH_REF" || true)"
+ PEELED="$(printf '%s\n' "$LS" | grep '\^{}$' | head -n1 | cut -f1)"
+ if [ -n "$PEELED" ]; then
+ RESOLVED="$PEELED"
+ else
+ RESOLVED="$(printf '%s\n' "$LS" | head -n1 | cut -f1)"
+ fi
+ if [ -n "$RESOLVED" ]; then break; fi
+ echo "ls-remote attempt $attempt resolved nothing"
+ sleep 10
+ done
+ fi
+ echo "head_sha=$RESOLVED" >> "$GITHUB_OUTPUT"
+ echo "the gate is keyed on unslothai/unsloth@${RESOLVED:-(unresolved)}"
+
- name: Decide
id: decide
env:
@@ -479,6 +522,9 @@ jobs:
run: |
python .github/scripts/kaggle_t4_ci/gate.py \
--percent 15 \
+ --head-sha '${{ steps.ref.outputs.head_sha }}' \
+ --kind notebook \
+ --slot '${{ inputs.slot || '1' }}' \
--force '${{ inputs.force || (github.event_name == 'schedule' && 'true' || 'false') }}' \
--labels "$PR_LABELS" \
--label-name kaggle-t4-ci \
@@ -493,6 +539,13 @@ jobs:
needs: gate
if: needs.gate.outputs.should_run == 'true'
runs-on: ubuntu-latest
+ # `statuses: write` is what makes the dispatch honest: this job no longer
+ # waits, so it cannot BE the check that says the GPU tests passed. It posts
+ # that verdict against the sha the result belongs to, which need not be this
+ # run's; the collection step below finishes earlier commits' kernels.
+ permissions:
+ contents: read
+ statuses: write
# A ONE-ELEMENT matrix, whose only job is to carry the account the gate
# chose into a context that `secrets[...]` can be indexed with.
#
@@ -738,31 +791,18 @@ jobs:
env:
UNSLOTH_REF: ${{ inputs.unsloth_ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ GATE_SHA: ${{ needs.gate.outputs.head_sha }}
run: |
if [ -z "$UNSLOTH_REF" ]; then
echo "ref=$HEAD_SHA" >> "$GITHUB_OUTPUT"
echo "testing unslothai/unsloth@$HEAD_SHA"
exit 0
fi
- if printf '%s' "$UNSLOTH_REF" | grep -Eq '^[0-9a-f]{40}$'; then
- RESOLVED="$UNSLOTH_REF"
- else
- RESOLVED=''
- for attempt in 1 2 3; do
- LS="$(git ls-remote https://github.com/unslothai/unsloth "$UNSLOTH_REF" || true)"
- # An annotated tag lists the tag OBJECT first and the commit it
- # points at on the `^{}` line, which is what pip can check out.
- PEELED="$(printf '%s\n' "$LS" | grep '\^{}$' | head -n1 | cut -f1)"
- if [ -n "$PEELED" ]; then
- RESOLVED="$PEELED"
- else
- RESOLVED="$(printf '%s\n' "$LS" | head -n1 | cut -f1)"
- fi
- if [ -n "$RESOLVED" ]; then break; fi
- echo "ls-remote attempt $attempt resolved nothing"
- sleep 10
- done
- fi
+ # RESOLVED ONCE, by the gate job, and reused rather than asked again:
+ # a branch or tag can move while this job queues, and a second answer
+ # could name a commit the gate never keyed the draw or the in-flight
+ # check on, whose own run may already be out on the other account.
+ RESOLVED="$GATE_SHA"
# Every branch above ends here, including the 40-character one. That
# branch used to be its own exit: a SHA is immutable, so it was
# accepted on shape alone and nothing asked whether unslothai/unsloth
@@ -1005,6 +1045,61 @@ jobs:
$SHARED_WHEELS \
$SKIP
+ # BEFORE THE RECHECK, finish what an earlier run started.
+ #
+ # Before and not after: this can take fifteen minutes of downloads, and a
+ # recheck on the far side of that gap is the stale answer it exists to
+ # replace, since the Studio workflow can take the account's last session
+ # meanwhile. Collecting first also frees whatever slot a finished kernel
+ # held.
+ #
+ # The cheap half of collection: this job already has a runner and this
+ # account's token, so a previous commit's finished kernel costs seconds
+ # and publishes against ITS OWN sha. `kaggle-collect.yml` does the same on
+ # a schedule, always but only every ten minutes.
+ #
+ # It also answers whether a kernel for THIS commit is already in flight,
+ # so a re-run does not buy a second session for a result already coming.
+ - name: Collect finished Kaggle runs
+ id: collect
+ if: steps.ref.outputs.stand_down != 'true' && steps.pins.outputs.stand_down != 'true' && steps.stepcount.outputs.stand_down != 'true'
+ env:
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/collect.py \
+ --outdir kaggle_collected \
+ --expect '${{ steps.build.outputs.payloads }}' \
+ --sha '${{ steps.ref.outputs.ref }}' \
+ --kind notebook \
+ --slot '${{ inputs.slot || '1' }}' \
+ --no-delete \
+ --target-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+
+ # Post FIRST, delete SECOND: a verdict that fails to reach GitHub leaves
+ # its kernel up for the next pass, and deleting first would make that
+ # failure permanent. The poster is shared by all three workflows; it
+ # expands the slug's abbreviated sha (the statuses API 422s one) and hands
+ # every field to `gh` as its own argument, never through a shell.
+ - name: Post the collected verdicts as commit statuses
+ id: post
+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/post_statuses.py \
+ --result kaggle_collected/collect_result.json \
+ --out kaggle_collected/posted.json
+
+ - name: Release the kernels whose verdicts were delivered
+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''
+ env:
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/collect.py \
+ --outdir kaggle_collected \
+ --delete-collected kaggle_collected/collect_result.json \
+ --posted kaggle_collected/posted.json
+
# kernel-timeout-sec is passed to Kaggle at push time as a BACKSTOP, not
# the budget control. Measured on 2026-08-11: a kernel pushed with -t 5400
# whose nbconvert crashed sat in RUNNING for over two hours and stopped
@@ -1015,6 +1110,11 @@ jobs:
# takes it per push and the larger requirement wins: kernel 2 downloads a
# 12GB gpt-oss checkpoint before it trains anything.
#
+ # `--sha` stands a run down when a kernel of this commit is already up,
+ # and `--slot` is part of that identity: slot 2 is the opt-in for a SECOND
+ # session on the same ref, so a slot-1 kernel does not count against it
+ # while a retry of the slot-2 run itself does.
+ #
# THE GATE'S ANSWER CAN BE STALE BY THE TIME WE GET HERE, and this is the
# first point at which it cannot be. The `gate` job surveys quota and
# in-flight kernels before this job exists, and this job then queues on
@@ -1064,6 +1164,11 @@ jobs:
run: |
echo "::warning title=Stood down at the second check::${REASON}"
+ - name: Note that this commit is already running on Kaggle
+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight == 'true'
+ run: |
+ echo "::notice title=Already in flight::a Kaggle kernel for this commit is still running from an earlier run, so nothing was dispatched. Its result will arrive as the kaggle-t4-notebook commit status."
+
# --deadline-epoch is when THIS JOB is killed, and the launcher pushes
# nothing unless its whole worst case still fits before it. Everything
# above is unbounded -- a checkout, a pip install off a slow index, the
@@ -1080,9 +1185,19 @@ jobs:
# JOB_TIMEOUT_MINUTES restates this job's timeout-minutes because a job
# cannot read its own; the harness suite asserts the two agree, so moving
# one without the other is a red test rather than a silent gap.
- - name: Launch on Kaggle and collect
+ #
+ # PUSH AND EXIT. Waiting here cost 41.5 minutes of runner per commit
+ # (measured, run 33479481067) against a queue 400-670 deep.
+ #
+ # This job now succeeds by DISPATCHING and proves only that Kaggle
+ # accepted a push. The verdict arrives later as the `kaggle-t4-notebook`
+ # commit status, which cannot be a REQUIRED check while the gate samples
+ # and the workflow path-filters: a skipped commit never gets one.
+ # --commit-sha puts the sha into the slug, the only record outliving this
+ # runner.
+ - name: Dispatch to Kaggle
id: launch
- if: steps.recheck.outputs.should_run == 'true'
+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight != 'true'
env:
KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
JOB_TIMEOUT_MINUTES: 260
@@ -1095,8 +1210,28 @@ jobs:
--kernel-timeout-sec 3600 \
--max-wait 5400 \
--poll-every 60 \
+ --dispatch \
+ --kind notebook \
+ --slot '${{ inputs.slot || '1' }}' \
+ --commit-sha '${{ steps.ref.outputs.ref }}' \
--deadline-epoch "$(( JOB_START_EPOCH + JOB_TIMEOUT_MINUTES * 60 ))"
+ # Gated on the launcher's verdict, not its exit code: launch.py exits 0
+ # for every stand-down, and a pending status for a kernel that does not
+ # exist is one no collector can ever replace.
+ - name: Mark the dispatch pending on this commit
+ if: steps.launch.outputs.verdict == 'dispatched'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ # A required context that has never been posted blocks a pull request
+ # forever with no explanation on the page.
+ gh api "repos/$GITHUB_REPOSITORY/statuses/${{ steps.ref.outputs.ref }}" \
+ -f state=pending -f context=kaggle-t4-notebook \
+ -f description="dispatched to Kaggle; the result is collected when the kernel finishes" \
+ -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+ --silent || echo "::warning title=Could not post the pending status::kaggle-t4-notebook"
+
# The launcher collects executed notebooks and the kernel log and nothing
# else, deliberately. The Studio payload therefore encodes its
# screenshots and logs into its own cell output; this puts them back.
@@ -1105,12 +1240,19 @@ jobs:
# It is safe on a run that carried no Studio payload: the collector finds
# no encoded evidence and writes nothing.
- name: Unpack the Playwright evidence
- if: always() && steps.recheck.outputs.should_run == 'true'
+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb', 'kaggle_collected/**/*_output.ipynb') != ''
continue-on-error: true
run: |
- python .github/scripts/kaggle_studio_ci/collect_evidence.py \
- --evidence kaggle_evidence \
- --outdir studio_evidence
+ # One kernel per call: the bundle is reassembled from numbered chunks
+ # and one walk over two would splice them together. Both trees, since
+ # a waited-on kernel lands in kaggle_evidence and a collected one in
+ # kaggle_collected.
+ for dir in kaggle_evidence/*/ kaggle_collected/*/; do
+ [ -d "$dir" ] || continue
+ python .github/scripts/kaggle_studio_ci/collect_evidence.py \
+ --evidence "$dir" \
+ --outdir "studio_evidence/$(basename "$dir")"
+ done
# always(): the evidence is most valuable when something went wrong, and
# the executed notebooks are kilobytes.
@@ -1136,6 +1278,7 @@ jobs:
name: kaggle-t4-evidence
path: |
kaggle_evidence/**
+ kaggle_collected/**
studio_evidence/**
kernel*.ipynb
# 14 rather than the repo's usual 7: this job runs on a minority of
@@ -1157,7 +1300,7 @@ jobs:
# recheck has no evidence, and "0 of 4 payloads" would read as a failure
# rather than the skip it is.
- name: Report
- if: always() && steps.recheck.outputs.should_run == 'true'
+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb') != ''
run: |
python .github/scripts/kaggle_t4_ci/report.py \
--evidence kaggle_evidence \
@@ -1175,7 +1318,7 @@ jobs:
# report set with `partial` rather than silence, so ungated it would print
# a PARTIAL section on every leg-list run about a payload it never ran.
- name: Report Studio
- if: always() && steps.recheck.outputs.should_run == 'true' && steps.build.outputs.studio == 'true'
+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb') != '' && steps.build.outputs.studio == 'true'
run: |
python .github/scripts/kaggle_studio_ci/report.py \
--evidence kaggle_evidence
diff --git a/.github/workflows/kaggle-t4-studio-gpu-ci.yml b/.github/workflows/kaggle-t4-studio-gpu-ci.yml
index 64bf7301409..952b5f7a7ec 100644
--- a/.github/workflows/kaggle-t4-studio-gpu-ci.yml
+++ b/.github/workflows/kaggle-t4-studio-gpu-ci.yml
@@ -276,7 +276,10 @@ jobs:
&& (github.event.action != 'labeled'
|| github.event.label.name == 'kaggle-studio-gpu-ci')
runs-on: ubuntu-latest
- timeout-minutes: 10
+ # 15, the worst case with two accounts: each one's authentication, username
+ # and quota read at the socket ceiling, one shared survey budget, and the
+ # call still in flight when it expires. The suite asserts the arithmetic.
+ timeout-minutes: 15
outputs:
should_run: ${{ steps.decide.outputs.should_run }}
reason: ${{ steps.decide.outputs.reason }}
@@ -286,6 +289,9 @@ jobs:
account: ${{ steps.decide.outputs.account }}
account_user: ${{ steps.decide.outputs.account_user }}
matrix: ${{ steps.decide.outputs.matrix }}
+ # The commit under test, resolved once: the GPU job tests THIS and never
+ # re-resolves a moving ref.
+ head_sha: ${{ steps.ref.outputs.head_sha }}
steps:
- name: Harden runner (audit)
uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0
@@ -311,14 +317,54 @@ jobs:
# demanding two free concurrency slots would stand the job down on an
# account that has room for it.
#
- # budget-hours is the worst case ONE invocation can cost: the 70-minute
- # kernel ceiling set below, rounded up.
+ # budget-hours is the worst case ONE invocation can cost. Nothing waits
+ # on the kernel now, and one that ignores its own timeout (seen: two hours
+ # past it, still RUNNING) is left to the collector's age ceiling plus its
+ # schedule and job timeout, so the worst case is that whole reaper window.
+ # The harness suite derives it.
#
# reserve-hours is 10 rather than the notebook leg's 20, and that is the
# single knob that implements "CI may use up to 50 of the 60 hours".
# Raising it throttles BOTH legs, since they read the same account-wide
# quota. Do not raise it above roughly 45 or CI never runs at all on a
# week with any other usage.
+ # THE COMMIT THE GPU JOB WILL TEST, which is not always this workflow's
+ # own: a dispatch naming `unsloth_ref` tests that ref. Keyed on github.sha
+ # instead, the in-flight check would look for the wrong kernel, and once
+ # the default branch moved two runs of one requested ref could be drawn
+ # onto different accounts and never see each other. A ref that does not
+ # resolve leaves the key EMPTY, so the draw falls back to the run id and
+ # the GPU job stands down on the same ref moments later.
+ - name: Name the commit under test
+ id: ref
+ env:
+ UNSLOTH_REF: ${{ inputs.unsloth_ref }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ run: |
+ if [ -z "$UNSLOTH_REF" ]; then
+ echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ if printf '%s' "$UNSLOTH_REF" | grep -Eq '^[0-9a-f]{40}$'; then
+ RESOLVED="$UNSLOTH_REF"
+ else
+ RESOLVED=''
+ for attempt in 1 2 3; do
+ LS="$(git ls-remote https://github.com/unslothai/unsloth "$UNSLOTH_REF" || true)"
+ PEELED="$(printf '%s\n' "$LS" | grep '\^{}$' | head -n1 | cut -f1)"
+ if [ -n "$PEELED" ]; then
+ RESOLVED="$PEELED"
+ else
+ RESOLVED="$(printf '%s\n' "$LS" | head -n1 | cut -f1)"
+ fi
+ if [ -n "$RESOLVED" ]; then break; fi
+ echo "ls-remote attempt $attempt resolved nothing"
+ sleep 10
+ done
+ fi
+ echo "head_sha=$RESOLVED" >> "$GITHUB_OUTPUT"
+ echo "the gate is keyed on unslothai/unsloth@${RESOLVED:-(unresolved)}"
+
- name: Decide
id: decide
env:
@@ -329,16 +375,25 @@ jobs:
run: |
python .github/scripts/kaggle_t4_ci/gate.py \
--percent 5 \
+ --head-sha '${{ steps.ref.outputs.head_sha }}' \
+ --kind studio \
--force '${{ inputs.force || 'false' }}' \
--labels '${{ join(github.event.pull_request.labels.*.name, ',') }}' \
--label-name kaggle-studio-gpu-ci \
--kernels 1 \
- --budget-hours 2 \
+ --budget-hours 4 \
--reserve-hours 10
studio-gpu:
name: Unsloth GPU smoke
needs: gate
+ # Load-bearing: this job dispatches rather than waits, so it cannot BE the
+ # check that says the Studio smoke passed. It posts that verdict as a status
+ # on whichever commit the collected kernel belongs to, not necessarily this
+ # run's.
+ permissions:
+ contents: read
+ statuses: write
if: needs.gate.outputs.should_run == 'true'
runs-on: ubuntu-latest
# One element, carrying the chosen account into a context `secrets[...]`
@@ -394,15 +449,55 @@ jobs:
# head SHA, so the kernel exercises the proposed code and not the merge
# base. Pinned to a SHA rather than a branch so a push landing mid-run
# cannot change what was tested.
+ # To a COMMIT, never a ref: the commit goes into the kernel's slug, the
+ # only record a later collector has. A branch or tag cannot be encoded,
+ # and launch.py refuses it.
- name: Resolve the ref under test
id: ref
+ env:
+ UNSLOTH_REF: ${{ inputs.unsloth_ref }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ GATE_SHA: ${{ needs.gate.outputs.head_sha }}
run: |
- REF='${{ inputs.unsloth_ref }}'
- if [ -z "$REF" ]; then
- REF='${{ github.event.pull_request.head.sha || github.sha }}'
+ if [ -z "$UNSLOTH_REF" ]; then
+ echo "ref=$HEAD_SHA" >> "$GITHUB_OUTPUT"
+ echo "testing unslothai/unsloth@$HEAD_SHA"
+ exit 0
+ fi
+ # RESOLVED ONCE, by the gate job, and reused rather than asked again:
+ # a branch or tag can move while this job queues, and a second answer
+ # could name a commit the gate never keyed the draw or the in-flight
+ # check on, whose own run may already be out on the other account.
+ RESOLVED="$GATE_SHA"
+ # A full SHA is accepted on shape alone above and `git ls-remote`
+ # matches refs, so neither says the repo HAS the commit. A mistyped or
+ # force-pushed-away SHA would spend a session and never get a verdict.
+ # Fetching the object is the reachability pip needs anyway; GitHub
+ # refuses an unreachable one with exit 128.
+ if ! printf '%s' "$RESOLVED" | grep -Eq '^[0-9a-f]{40}$'; then
+ RESOLVED=''
fi
- echo "ref=$REF" >> "$GITHUB_OUTPUT"
- echo "testing unslothai/unsloth@$REF"
+ if [ -n "$RESOLVED" ]; then
+ EXISTS=''
+ for attempt in 1 2 3; do
+ if git fetch --quiet --depth=1 https://github.com/unslothai/unsloth "$RESOLVED"; then
+ EXISTS=yes
+ break
+ fi
+ echo "fetch attempt $attempt did not find $RESOLVED"
+ sleep 10
+ done
+ if [ -z "$EXISTS" ]; then
+ echo "::error title=The requested unsloth commit does not exist::unslothai/unsloth does not serve the commit $RESOLVED, so a session would be spent on a commit no status can ever be posted to."
+ exit 1
+ fi
+ fi
+ if [ -z "$RESOLVED" ]; then
+ echo "::error title=Could not resolve the requested unsloth ref::'$UNSLOTH_REF' names no branch, tag or full commit on unslothai/unsloth, so there is no commit to dispatch or to report against."
+ exit 1
+ fi
+ echo "ref=$RESOLVED" >> "$GITHUB_OUTPUT"
+ echo "testing unslothai/unsloth@$RESOLVED (from '$UNSLOTH_REF')"
- name: Build the kernel notebook
env:
@@ -428,6 +523,50 @@ jobs:
--payload-args "$ARGS" \
--per-run-timeout 3900
+ # Finish what an earlier run started, BEFORE the recheck: this can spend
+ # fifteen minutes downloading and the notebook workflow can take the
+ # account's last session in that window; the recheck must be the last
+ # thing before the push. As on the notebook leg, the token is already in
+ # hand, collecting frees the slot this job wants, and it answers whether a
+ # kernel for THIS commit is already in flight.
+ - name: Collect finished Kaggle runs
+ id: collect
+ env:
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/collect.py \
+ --outdir kaggle_collected \
+ --expect 1 \
+ --sha '${{ steps.ref.outputs.ref }}' \
+ --kind studio \
+ --no-delete \
+ --target-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+
+ # Post FIRST, delete SECOND: a verdict that fails to reach GitHub leaves
+ # its kernel up for the next pass, and deleting first would make that
+ # failure permanent. The poster is shared by all three workflows; it
+ # expands the slug's abbreviated sha (the statuses API 422s one) and hands
+ # every field to `gh` as its own argument, never through a shell.
+ - name: Post the collected verdicts as commit statuses
+ id: post
+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/post_statuses.py \
+ --result kaggle_collected/collect_result.json \
+ --out kaggle_collected/posted.json
+
+ - name: Release the kernels whose verdicts were delivered
+ if: always() && hashFiles('kaggle_collected/collect_result.json') != ''
+ env:
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/collect.py \
+ --outdir kaggle_collected \
+ --delete-collected kaggle_collected/collect_result.json \
+ --posted kaggle_collected/posted.json
+
# WITH THE ACCOUNT SLOT IN HAND, and immediately before the push. The
# notebook leg has always done this and this one never has; two accounts
# is what makes the gap matter. A gate answer can be a whole run stale by
@@ -450,7 +589,7 @@ jobs:
--soft-fail \
--account-env KAGGLE_API_TOKEN \
--kernels 1 \
- --budget-hours 2 \
+ --budget-hours 4 \
--reserve-hours 10
# Through the environment rather than an inline expression, so nothing the
@@ -462,6 +601,11 @@ jobs:
run: |
echo "::warning title=Stood down at the second check::${REASON}"
+ - name: Note that this commit is already running on Kaggle
+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight == 'true'
+ run: |
+ echo "::notice title=Already in flight::a Kaggle kernel for this commit is still running from an earlier run, so nothing was dispatched. Its result will arrive as the kaggle-studio-gpu commit status."
+
# kernel-timeout-sec is the load-bearing budget control: Kaggle itself
# kills the session at that point. A runner that is cancelled cannot
# clean up after itself, so the ceiling has to be enforced on Kaggle's
@@ -470,12 +614,18 @@ jobs:
# The launcher is shared with the notebook leg and is used unchanged:
# this payload prints the same T4_SMOKE_REPORT line, so the transport
# needed no modification.
- - name: Launch on Kaggle and collect
+ #
+ # PUSH AND EXIT. Waiting here cost 18.7 minutes of runner (run
+ # 33486360729). This job now succeeds by dispatching and cannot say
+ # whether the Studio smoke passed; the verdict arrives as the
+ # `kaggle-studio-gpu` commit status. Not a required check while the gate
+ # samples: a skipped commit never gets one.
+ - name: Dispatch to Kaggle
id: launch
# GATED ON THE RECHECK, or the recheck is decorative: a step that
# measures the account and cannot stop the push is a log line, not a
# control. This is the same shape the notebook leg uses.
- if: steps.recheck.outputs.should_run == 'true'
+ if: steps.recheck.outputs.should_run == 'true' && steps.collect.outputs.in_flight != 'true'
env:
KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
run: |
@@ -486,17 +636,46 @@ jobs:
--expect 1 \
--kernel-timeout-sec 4200 \
--max-wait 5400 \
- --poll-every 60
+ --poll-every 60 \
+ --dispatch \
+ --kind studio \
+ --commit-sha '${{ steps.ref.outputs.ref }}'
+
+ # Gated on the launcher's verdict, not its exit code: launch.py exits 0
+ # for every stand-down, and a pending status for a kernel that does not
+ # exist is one no collector can ever replace.
+ - name: Mark the dispatch pending on this commit
+ if: steps.launch.outputs.verdict == 'dispatched'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ # A pending status as soon as the kernel is queued, so a required
+ # context that has not reported yet reads as waiting, not as nothing.
+ gh api "repos/$GITHUB_REPOSITORY/statuses/${{ steps.ref.outputs.ref }}" \
+ -f state=pending -f context=kaggle-studio-gpu \
+ -f description="dispatched to Kaggle; the result is collected when the kernel finishes" \
+ -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+ --silent || echo "::warning title=Could not post the pending status::kaggle-studio-gpu"
# The launcher collects executed notebooks and the kernel log and
# nothing else, deliberately. The payload therefore encodes its
# screenshots and logs into its own cell output; this puts them back.
- name: Unpack the Playwright evidence
- if: always()
+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb', 'kaggle_collected/**/*_output.ipynb') != ''
+ # Best effort: the verdict was posted above, and evidence Kaggle
+ # mangled must not turn the job red after the fact.
+ continue-on-error: true
run: |
- python .github/scripts/kaggle_studio_ci/collect_evidence.py \
- --evidence kaggle_evidence \
- --outdir studio_evidence
+ # One kernel per call: the bundle is reassembled from numbered chunks
+ # and one walk over two would splice them together. Both trees, since
+ # a waited-on kernel lands in kaggle_evidence and a collected one in
+ # kaggle_collected.
+ for dir in kaggle_evidence/*/ kaggle_collected/*/; do
+ [ -d "$dir" ] || continue
+ python .github/scripts/kaggle_studio_ci/collect_evidence.py \
+ --evidence "$dir" \
+ --outdir "studio_evidence/$(basename "$dir")"
+ done
# always(): the evidence is most valuable exactly when something went
# wrong. Screenshots only travel on a failing run, so a passing upload
@@ -508,6 +687,7 @@ jobs:
name: kaggle-studio-gpu-evidence
path: |
kaggle_evidence/**
+ kaggle_collected/**
studio_evidence/**
kernel.ipynb
# 14 rather than the repo's usual 7. This job runs on a minority of
@@ -517,7 +697,7 @@ jobs:
if-no-files-found: warn
- name: Report
- if: always()
+ if: always() && hashFiles('kaggle_evidence/**/*_output.ipynb') != ''
run: |
python .github/scripts/kaggle_studio_ci/report.py \
--evidence kaggle_evidence
diff --git a/.gitignore b/.gitignore
index 5f406be9fae..1097bf4af7f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -254,3 +254,6 @@ Speech_12dB_s16.flac
# storage_roots.studio_root at it. It is a mutable SQLite runtime database, not a fixture,
# so a test run or an Unsloth start rewrites it and dirties the checkout.
.studio-test-root/
+
+# launch.py --outdir scratch from a local run
+/out/
diff --git a/tests/kaggle/test_fire_and_forget.py b/tests/kaggle/test_fire_and_forget.py
new file mode 100644
index 00000000000..35e3240c60a
--- /dev/null
+++ b/tests/kaggle/test_fire_and_forget.py
@@ -0,0 +1,1791 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Dispatch and collect: the ways this trades forty minutes for nothing.
+
+The GPU jobs no longer wait for their kernel, buying back 41.5 and 18.7 minutes
+of runner per commit (runs 33479481067 and 33486360729). Every test here aims
+at the one new class of bug, **the result stops arriving and nothing says so**,
+in its four green shapes:
+
+1. the dispatching job reports success and is still the required check, so the
+ check now means "Kaggle accepted a push";
+2. the kernel is never collected, so it bills quota to its ceiling and the
+ commit keeps a status that never resolves;
+3. the collector deletes a kernel before reading it, or one never ours;
+4. the slug loses the commit, so a finished result cannot be attributed.
+
+Everything runs on CPU against stubs; no Kaggle session is spent.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+CI_DIR = REPO_ROOT / ".github" / "scripts" / "kaggle_t4_ci"
+WORKFLOWS = REPO_ROOT / ".github" / "workflows"
+NOTEBOOK_WF = WORKFLOWS / "kaggle-t4-notebook-ci.yml"
+STUDIO_WF = WORKFLOWS / "kaggle-t4-studio-gpu-ci.yml"
+COLLECT_WF = WORKFLOWS / "kaggle-collect.yml"
+
+sys.path.insert(0, str(CI_DIR))
+
+import collect # noqa: E402
+import gate # noqa: E402
+import launch # noqa: E402
+import post_statuses # noqa: E402
+
+
+def _wf(path: Path) -> dict:
+ return yaml.safe_load(path.read_text(encoding = "utf-8"))
+
+
+def _steps(workflow: dict) -> list[tuple[str, str, dict]]:
+ out = []
+ for job_name, job in workflow["jobs"].items():
+ for step in job.get("steps", []) or []:
+ out.append((job_name, step.get("name", ""), step))
+ return out
+
+
+# ------------------------------------------------------- the slug is the record
+
+
+def test_the_slug_carries_the_commit_and_the_workflow():
+ """The slug is the ONLY thing surviving the dispatching runner: without the
+ commit and the workflow in it, a finished kernel has nowhere to report."""
+ name = launch.slug_name("notebook", "1a2b3c4d5e6f7890")
+ parsed = launch.parse_slug(name)
+ assert parsed is not None, f"{name} does not parse as one of ours"
+ assert parsed["sha"] == "1a2b3c4d5e6f", parsed
+ assert parsed["kind"] == "notebook", parsed
+
+
+def test_the_slug_carries_twelve_hex_characters_and_still_reads_eight():
+ """Two commits can share eight characters, and the commits API 422s the
+ ambiguous prefix on every pass, so the status stays pending forever. Twelve
+ is written; eight is still read, since old kernels outlive the change."""
+ assert launch.SLUG_SHA_LEN == 12
+ name = launch.slug_name("studio", "a" * 40)
+ assert launch.parse_slug(name)["sha"] == "a" * 12
+ old = launch.parse_slug("me/unsloth-t4-ci-nabcdef01-1111")
+ assert old is not None and old["sha"] == "abcdef01" and old["kind"] == "notebook"
+ # Too short to be unambiguous is not written into a slug at all: it falls
+ # back to the unattributable form rather than inventing a prefix.
+ assert launch.parse_slug(launch.slug_name("notebook", "abcdef01"))["legacy"] is True
+
+
+def test_in_flight_matches_the_old_and_the_new_slug_forms(tmp_path, monkeypatch):
+ """A kernel pushed with the eight-character form must still count as in
+ flight for its full commit, and a twelve-character one for the same."""
+ full = "abcdef0123456789" + "0" * 24
+ for slug_sha in ("abcdef01", "abcdef012345"):
+ slug = f"danielhanchen/unsloth-t4-ci-n{slug_sha}-1111"
+ api = _StubApi([_StubKernel(slug)], {slug: "RUNNING"})
+ monkeypatch.setattr(launch, "_api", lambda api = api: api)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ ["collect", "--outdir", str(tmp_path / slug_sha), "--sha", full, "--kind", "notebook"],
+ )
+ assert collect.main() == 0
+ result = json.loads((tmp_path / slug_sha / "collect_result.json").read_text())
+ assert result["in_flight_for_sha"] is True, (slug_sha, result)
+
+
+def test_the_slug_carries_the_slot_and_reads_slot_one_when_absent():
+ """Slot 2 is a deliberate second session beside slot 1, so the duplicate
+ check needs "this commit, this kind, THIS slot" and the slug has to carry
+ the slot. Absent means slot 1, so every earlier slug still reads."""
+ two = launch.slug_name("notebook", "1a2b3c4d5e6f7890", slot = "2")
+ parsed = launch.parse_slug(two)
+ assert parsed and parsed["slot"] == "2" and parsed["sha"] == "1a2b3c4d5e6f", (two, parsed)
+ one = launch.slug_name("notebook", "1a2b3c4d5e6f7890", slot = "1")
+ assert launch.parse_slug(one)["slot"] == "1"
+ assert launch.parse_slug("me/unsloth-t4-ci-n1a2b3c4d5e6f-abcd")["slot"] == "1"
+ assert launch.parse_slug("me/unsloth-t4-ci-nabcdef01-1111")["slot"] == "1"
+ assert launch.parse_slug("me/unsloth-t4-ci-nabcdef01-21111")["slot"] == "2"
+ # Anything that is not a single digit 2-9 is slot 1 in the slug.
+ assert launch.parse_slug(launch.slug_name("notebook", "1a2b3c4d5e6f", slot = "23"))["slot"] == "1"
+ assert launch._slugify(two.replace("-", " ")) == two
+ # The collector narrows its in-flight answer by slot too.
+ assert "slot" in launch.parse_slug(two)
+ assert (
+ gate.in_flight_for_commit([f"me/{two} (RUNNING)"], "1a2b3c4d5e6f7890", "notebook", "2")
+ == f"me/{two}"
+ )
+ assert (
+ gate.in_flight_for_commit([f"me/{two} (RUNNING)"], "1a2b3c4d5e6f7890", "notebook", "1")
+ is None
+ )
+ assert (
+ gate.in_flight_for_commit([f"me/{one} (RUNNING)"], "1a2b3c4d5e6f7890", "notebook", "2")
+ is None
+ )
+
+
+def test_the_collector_in_flight_answer_is_per_slot(tmp_path, monkeypatch):
+ full = "abcdef0123456789" + "0" * 24
+ slug = "danielhanchen/unsloth-t4-ci-nabcdef012345-21111"
+ for slot, expected in (("2", True), ("1", False)):
+ api = _StubApi([_StubKernel(slug)], {slug: "RUNNING"})
+ monkeypatch.setattr(launch, "_api", lambda api = api: api)
+ out = tmp_path / f"slot{slot}"
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ ["collect", "--outdir", str(out), "--sha", full, "--kind", "notebook", "--slot", slot],
+ )
+ assert collect.main() == 0
+ result = json.loads((out / "collect_result.json").read_text())
+ assert result["in_flight_for_sha"] is expected, (slot, result)
+
+
+def test_two_dispatches_of_one_commit_do_not_collide():
+ """A re-run, or slots 1 and 2, must not push to one slug: pushing to an
+ existing id starts a second session the status calls never answer for."""
+ names = {launch.slug_name("notebook", "deadbeefcafe") for _ in range(200)}
+ assert len(names) > 190, f"only {len(names)} distinct slugs in 200 draws"
+
+
+def test_the_slug_still_round_trips_through_slugify():
+ """Kaggle files the kernel under the slugified TITLE, not the metadata id,
+ so a name that does not round-trip 403s every later status call."""
+ for kind in ("notebook", "studio"):
+ name = launch.slug_name(kind, "0123456789ab")
+ assert launch._slugify(name.replace("-", " ")) == name, name
+
+
+def test_the_gate_still_recognises_a_dispatched_kernel_as_ours():
+ """THE ONE THAT KEEPS THE CONCURRENCY CONTROL WORKING. The GitHub group no
+ longer bounds Kaggle sessions; the gate's survey of slugs carrying
+ OWN_KERNEL_PREFIX does, and without the prefix every push believes the
+ account idle until Kaggle refuses one at its 2-session cap."""
+ import gate
+ for kind in ("notebook", "studio"):
+ name = launch.slug_name(kind, "abcdef01")
+ assert name.startswith(gate.OWN_KERNEL_PREFIX), (
+ f"{name} does not start with the gate's {gate.OWN_KERNEL_PREFIX!r}, so the "
+ "gate would count a dispatched kernel as somebody else's or not at all"
+ )
+
+
+ "foreign",
+ (
+ "danielhanchen/unsloth-probe-vision-train-r3-6dd742",
+ "someone/my-important-notebook",
+ "danielhanchen/unsloth-t4-cixyz",
+ "danielhanchen/unsloth-studio-ci-12345678",
+ "",
+ ),
+)
+def test_a_kernel_that_is_not_ours_is_invisible(foreign):
+ """The collector enumerates a whole ACCOUNT shared with a human and DELETES
+ what it collects, so there is deliberately no heuristic: a false positive
+ deletes somebody's work and looks exactly like a kernel that finished."""
+ assert launch.parse_slug(foreign) is None, f"{foreign!r} was accepted as one of ours"
+
+
+def test_a_legacy_slug_is_still_reapable_but_reports_nothing():
+ """Kernels pushed before this change outlive it: still ours and reapable, or
+ they bill forever, but with no commit to attribute a verdict to."""
+ parsed = launch.parse_slug("danielhanchen/unsloth-t4-ci-d7faf2b8")
+ assert parsed is not None and parsed["legacy"] is True
+ assert parsed["sha"] is None and parsed["kind"] is None
+ assert collect.statuses_from([{**parsed, "slug": "x", "verdict": "pass", "reason": "ok"}]) == []
+
+
+# ------------------------------------------------------------ dispatch refuses
+
+
+def test_dispatch_without_a_commit_is_a_usage_error(monkeypatch, capsys):
+ """A dispatch whose slug carries no commit runs, costs quota, and reports to
+ nobody. Refused at the only moment it is still free."""
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ ["launch.py", "--notebook", "k.ipynb", "--outdir", "out", "--dispatch"],
+ )
+ with pytest.raises(SystemExit) as excinfo:
+ launch.main()
+ assert excinfo.value.code != 0
+ assert "commit-sha" in capsys.readouterr().err
+
+
+def test_dispatch_does_not_delete_the_kernel_it_pushed():
+ """The whole point: the kernel is LEFT RUNNING, through the existing
+ --keep-kernel flag rather than a second condition in release(). Two guards
+ on one deletion is how one ends up wrong, and this one bills GPU quota."""
+ src = (CI_DIR / "launch.py").read_text(encoding = "utf-8")
+ assert re.search(r"if args\.dispatch:\s*\n\s*args\.keep_kernel = True", src), (
+ "dispatch mode must set keep_kernel, or release() deletes the kernel it "
+ "was told to leave running and the collector finds nothing"
+ )
+
+
+def test_dispatch_does_not_report_pass():
+ """`dispatched` is not `pass`, and the difference is the whole change.
+
+ Nothing has run when the dispatch returns, so `pass` would be worse than the
+ silent skip this repo has been caught by twice: it looks like a result.
+ """
+ src = (CI_DIR / "launch.py").read_text(encoding = "utf-8")
+ block = src[src.index("if args.dispatch:", src.index('result["slug"] = live[0]')) :]
+ block = block[: block.index("return finish()")]
+ assert '"dispatched"' in block, block[:400]
+ assert 'result["verdict"] = "pass"' not in block
+
+
+def test_the_dispatch_worst_case_excludes_the_phases_it_never_runs():
+ """Otherwise the pre-push guard demands two hours of job deadline for a job
+ that exits in five minutes, and stands runs down for a window they do not
+ need -- the opposite of what dispatch mode is for."""
+ full = launch.worst_case_seconds(5400, 1)
+ quick = launch.worst_case_seconds(5400, 1, dispatch = True)
+ assert quick < full, (quick, full)
+
+
+# ---------------------------------------------------------------- the collector
+
+
+class _StubKernel:
+ def __init__(
+ self,
+ ref,
+ last_run_time = None,
+ ):
+ self.ref = ref
+ self.last_run_time = last_run_time
+
+
+class _StubApi:
+ CONFIG_NAME_USER = "username"
+
+ def __init__(self, kernels, statuses):
+ self.config_values = {"username": "danielhanchen"}
+ self._kernels = kernels
+ self._statuses = statuses
+ self.status_calls: list[str] = []
+
+ def kernels_list(
+ self,
+ mine = True,
+ page = 1,
+ page_size = 100,
+ sort_by = "dateRun",
+ ):
+ return self._kernels if page == 1 else []
+
+ def kernels_status(self, ref):
+ self.status_calls.append(ref)
+ state = self._statuses[ref]
+ if isinstance(state, Exception):
+ raise state
+ return type("S", (), {"status": f"KernelWorkerStatus.{state}"})()
+
+
+def test_a_running_kernel_within_its_ceiling_is_left_completely_alone(tmp_path):
+ """No delete, no status, not even a warning: reporting a kernel doing its
+ job puts a red on a commit whose result is still coming."""
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "RUNNING"})
+ deleted: list[str] = []
+ entry = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 0.5,
+ }
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "pending"
+ assert record["deleted"] is False
+ assert collect.statuses_from([record]) == [], "a running kernel must post no status"
+ assert deleted == []
+
+
+def test_a_kernel_past_its_ceiling_is_reaped_and_reported(tmp_path, monkeypatch):
+ """THE REASON THE SCHEDULED COLLECTOR EXISTS. Nothing deletes the kernel
+ now, and a wedged one bills to its ceiling unwatched: one here was measured
+ ignoring Kaggle's own `-t` timeout for over two hours. Reported as a failure
+ rather than dropped, or the commit stays pending forever."""
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "RUNNING"})
+ entry = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 9.0,
+ }
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "reaped"
+ assert deleted == ["me/unsloth-t4-ci-nabcdef01-1111"]
+ status = collect.statuses_from([record])[0]
+ assert status["state"] == "failure", status
+
+
+def test_a_kernel_with_no_timestamp_is_never_reaped(tmp_path, monkeypatch):
+ """A missing timestamp is not evidence that a kernel is old, and guessing in
+ that direction DELETES A RUNNING SESSION."""
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "RUNNING"})
+ entry = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": None,
+ }
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "pending"
+ assert deleted == []
+
+
+def test_evidence_is_downloaded_before_the_kernel_is_deleted(tmp_path, monkeypatch):
+ """A delete that lands first turns a finished run into a result nobody can
+ read. Asserted by ORDER of the real calls, not by reading the source."""
+ order: list[str] = []
+ monkeypatch.setattr(
+ launch, "fetch_evidence", lambda slug, dest, deadline = None: order.append("fetch") or {}
+ )
+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: order.append("delete") or True
+ )
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})
+ entry = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 0.4,
+ }
+ collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)
+ assert order == ["fetch", "delete"], order
+
+
+def test_a_kernel_whose_evidence_will_not_download_is_NOT_deleted(tmp_path, monkeypatch):
+ """Deleting here destroys the only copy of a finished run's result to
+ reclaim a slot the kernel is not using. The next pass retries."""
+
+ def _boom(
+ slug,
+ dest,
+ deadline = None,
+ ):
+ raise TimeoutError("slow")
+
+ deleted: list[str] = []
+ monkeypatch.setattr(launch, "fetch_evidence", _boom)
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})
+ entry = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 0.4,
+ }
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)
+ # `pending`, not `infra`: an infra verdict posts green and is released by
+ # --delete-collected, breaking the "next pass retries" promise.
+ assert record["verdict"] == "pending"
+ assert record["verdict"] not in collect.DELETABLE
+ assert collect.statuses_from([record]) == []
+ assert deleted == [], "evidence that would not download must not be deleted"
+
+
+def test_an_unreadable_status_does_nothing_at_all(tmp_path, monkeypatch):
+ """Both available actions are destructive: delete and we may kill a running
+ session, report and we may fail a run that was fine. Ask again next pass."""
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": RuntimeError("503 upstream")})
+ entry = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 0.4,
+ }
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "pending"
+ assert deleted == []
+ assert collect.statuses_from([record]) == []
+
+
+def test_find_ours_skips_everything_it_does_not_recognise():
+ """Driven against a listing mixing a human's kernels with ours: a rule fed a
+ hand-written list of slugs proves nothing about the filter."""
+ listing = [
+ _StubKernel("danielhanchen/unsloth-probe-vision-train-r3-6dd742"),
+ _StubKernel("danielhanchen/unsloth-t4-ci-nabcdef01-1111"),
+ _StubKernel("danielhanchen/my-actual-work"),
+ _StubKernel("danielhanchen/unsloth-t4-ci-d7faf2b8"),
+ ]
+ api = _StubApi(listing, {})
+ ours = [k["slug"] for k in collect.find_ours(api)]
+ assert ours == [
+ "danielhanchen/unsloth-t4-ci-nabcdef01-1111",
+ "danielhanchen/unsloth-t4-ci-d7faf2b8",
+ ], ours
+
+
+def test_infra_and_partial_do_not_go_red():
+ """`infra` and `partial` learned nothing about the code, and a red the
+ author cannot act on is how a required check gets REMOVED."""
+ assert collect.VERDICT_STATE["infra"] == "success"
+ assert collect.VERDICT_STATE["partial"] == "success"
+ assert collect.VERDICT_STATE["fail"] == "failure"
+ assert collect.VERDICT_STATE["reaped"] == "failure"
+
+
+def test_a_failed_payload_reaches_github_as_a_failure():
+ """Driven through the real verdict and status functions, since every rule
+ above this one is fed a dict written by hand."""
+ record = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "verdict": None,
+ "reason": "",
+ }
+ verdict, reason = collect.verdict_of([{"passed": False, "payload": "gptoss"}], 1)
+ record["verdict"], record["reason"] = verdict, reason
+ status = collect.statuses_from([record])[0]
+ assert status["state"] == "failure"
+ assert status["context"] == "kaggle-t4-notebook"
+ assert "gptoss" in status["description"]
+
+
+def test_the_collector_never_sees_a_github_token():
+ """A script holding both a Kaggle and a GitHub credential is one bug away
+ from sending one to the other. It emits statuses as DATA; the workflow
+ posts them."""
+ src = (CI_DIR / "collect.py").read_text(encoding = "utf-8")
+ for forbidden in ("GH_TOKEN", "GITHUB_TOKEN", "api.github.com"):
+ assert forbidden not in src, f"collect.py references {forbidden}"
+
+
+# ------------------------------------------------------------- the workflows
+
+
+ "path,kind,context",
+ ((NOTEBOOK_WF, "notebook", "kaggle-t4-notebook"), (STUDIO_WF, "studio", "kaggle-studio-gpu")),
+ ids = ("notebook", "studio"),
+)
+def test_the_gpu_job_dispatches_rather_than_waits(path, kind, context):
+ """The measured win, asserted from the workflow rather than from a comment."""
+ body = path.read_text(encoding = "utf-8")
+ launch_calls = re.findall(r"kaggle_t4_ci/launch\.py \\\n(?:.*\n)*?(?=\n)", body)
+ assert launch_calls, f"{path.name} does not invoke launch.py"
+ for call in launch_calls:
+ assert "--dispatch" in call, f"{path.name} still waits for its kernel:\n{call}"
+ assert "--commit-sha" in call, f"{path.name} dispatches without a commit:\n{call}"
+ assert f"--kind {kind}" in call, f"{path.name} dispatches without its kind:\n{call}"
+
+
+ "path,context",
+ ((NOTEBOOK_WF, "kaggle-t4-notebook"), (STUDIO_WF, "kaggle-studio-gpu")),
+ ids = ("notebook", "studio"),
+)
+def test_the_dispatching_job_can_post_the_status_that_replaces_it(path, context):
+ """The job succeeds by dispatching, so the verdict has to travel some other
+ way. Without `statuses: write` the collection runs and reaches nobody."""
+ wf = _wf(path)
+ gpu_jobs = [j for name, j in wf["jobs"].items() if name != "gate"]
+ assert gpu_jobs, path.name
+ for job in gpu_jobs:
+ perms = job.get("permissions") or {}
+ assert perms.get("statuses") == "write", (
+ f"{path.name}'s GPU job cannot post commit statuses, so the verdict "
+ "produced by its collection step is discarded"
+ )
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_the_gpu_job_collects_before_it_dispatches(path):
+ """Order matters twice: collecting frees the session slot this job wants,
+ and it answers whether this commit is already running."""
+ body = path.read_text(encoding = "utf-8")
+ assert "kaggle_t4_ci/collect.py" in body, f"{path.name} never collects"
+ assert body.index("kaggle_t4_ci/collect.py") < body.index(
+ "--dispatch"
+ ), f"{path.name} dispatches before it collects"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_a_commit_already_in_flight_is_not_dispatched_again(path):
+ """Two sessions for one result is quota spent twice, and under Kaggle's
+ 2-session cap the second may take the slot the first needs."""
+ for _job, name, step in _steps(_wf(path)):
+ if "--dispatch" in (step.get("run") or ""):
+ assert "in_flight != 'true'" in (step.get("if") or ""), (
+ f"{path.name}'s dispatch step is not gated on the in-flight check: "
+ f"{step.get('if')!r}"
+ )
+ return
+ pytest.fail(f"{path.name} has no dispatch step")
+
+
+def test_a_scheduled_collector_exists_and_reaps():
+ """The other route only fires when somebody pushes, so a quiet weekend with
+ a wedged kernel is a weekend of unwatched billing."""
+ wf = _wf(COLLECT_WF)
+ on = wf.get(True) or wf.get("on")
+ assert "schedule" in on, "the collector does not run on a schedule, so a quiet repo never reaps"
+ assert on["schedule"], on
+
+
+def test_the_collector_covers_both_accounts():
+ """The gate's weighted draw decides where a session is SPENT; it must not
+ decide where results are read from."""
+ wf = _wf(COLLECT_WF)
+ job = wf["jobs"]["collect"]
+ names = {e["secret_name"] for e in job["strategy"]["matrix"]["include"]}
+ import gate
+
+ assert names == set(gate.DEFAULT_ACCOUNT_ENVS), names
+
+
+def test_the_collector_holds_one_token_per_job():
+ """As in the GPU workflows: a step holding both could authenticate as
+ either, and delete kernels for an account it is not reporting for."""
+ for _job, name, step in _steps(_wf(COLLECT_WF)):
+ env = step.get("env") or {}
+ tokens = sorted(k for k in env if k.startswith("KAGGLE_API_TOKEN"))
+ if tokens:
+ assert tokens == ["KAGGLE_API_TOKEN"], f"{name} sees {tokens}"
+
+
+def test_the_collector_serialises_per_account():
+ """Two collectors on one account race: the loser deletes what the winner
+ already deleted and reports the 404 as a failure."""
+ wf = _wf(COLLECT_WF)
+ conc = wf.get("concurrency")
+ assert conc, "the collector has no concurrency group"
+ assert conc.get("cancel-in-progress") is False, (
+ "a pass midway through downloading evidence must finish; cancelling it "
+ "throws away work the next pass has to redo"
+ )
+
+
+def test_the_status_contexts_are_stable_strings():
+ """Branch protection is configured against these names, and renaming one
+ silently stops requiring it: a context that never reports is not failing."""
+ assert collect.STATUS_CONTEXTS == {
+ "notebook": "kaggle-t4-notebook",
+ "studio": "kaggle-studio-gpu",
+ }
+ for path, context in ((NOTEBOOK_WF, "kaggle-t4-notebook"), (STUDIO_WF, "kaggle-studio-gpu")):
+ assert context in path.read_text(encoding = "utf-8"), (
+ f"{path.name} never mentions {context}, so its pending status is posted "
+ "under a name nothing else uses"
+ )
+
+
+def test_a_dispatch_posts_a_pending_status():
+ """Without it a required context that has never reported blocks the pull
+ request with no explanation, indistinguishable from an unconfigured check."""
+ for path, context in ((NOTEBOOK_WF, "kaggle-t4-notebook"), (STUDIO_WF, "kaggle-studio-gpu")):
+ body = path.read_text(encoding = "utf-8")
+ assert re.search(
+ r"state=pending -f context=" + re.escape(context), body
+ ), f"{path.name} never posts a pending {context} status"
+
+
+def test_the_reporters_wait_for_an_EXECUTED_NOTEBOOK_not_just_a_directory():
+ """Measured on run 33628507954, which reported `Kaggle T4 smoke: PARTIAL`
+ for a dispatch where nothing had run: `hashFiles('kaggle_evidence/**')` is
+ true once launch.py writes launch_result.json. The condition has to name
+ what a report is MADE of, an executed notebook."""
+ for path in (NOTEBOOK_WF, STUDIO_WF):
+ for _job, name, step in _steps(_wf(path)):
+ body = step.get("run") or ""
+ if "report.py" not in body:
+ continue
+ condition = step.get("if") or ""
+ assert "_output.ipynb" in condition, (
+ f"{path.name}: step {name!r} reports on the presence of the evidence "
+ f"DIRECTORY, which a dispatching run creates and does not fill: "
+ f"if={condition!r}"
+ )
+
+
+def _fake_gh(
+ monkeypatch,
+ resolve_to = None,
+ post_ok = True,
+ lookup_error = "",
+):
+ """Stand in for `gh`, recording every call. `resolve_to` is what
+ `repos/../commits/<sha>` answers, None means gone in GitHub's own words, and
+ `lookup_error` is what a failed lookup says."""
+ calls: list[list[str]] = []
+
+ def _gh(args):
+ calls.append(list(args))
+ if "/commits/" in args[1]:
+ if lookup_error:
+ return (1, "", lookup_error)
+ if resolve_to:
+ return (0, resolve_to, "")
+ return (1, "", "gh: No commit found for SHA: 2ecb19df (HTTP 422)")
+ return (0 if post_ok else 1, "", "")
+
+ monkeypatch.setattr(post_statuses, "_gh", _gh)
+ return calls
+
+
+_STATUS = {
+ "sha": "2ecb19df",
+ "state": "success",
+ "context": "kaggle-t4-notebook",
+ "description": "pass: all 5 payload(s) passed",
+ "target_url": "https://example/run/1",
+ "slug": "me/unsloth-t4-ci-n2ecb19df-1111",
+ "slugs": ["me/unsloth-t4-ci-n2ecb19df-1111"],
+}
+
+
+def test_an_abbreviated_sha_is_EXPANDED_before_a_status_is_posted(monkeypatch):
+ """MEASURED AGAINST THE REAL API, and it fails closed in the worst way:
+
+ POST /repos/{o}/{r}/statuses/2ecb19df
+ 422 "Sha must be a valid hex object ID"
+
+ The slug carries an abbreviation, so the poster must expand it first or
+ every verdict 422s while the collection quietly succeeds.
+ """
+ full = "2ecb19df" + "a" * 32
+ calls = _fake_gh(monkeypatch, resolve_to = full)
+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")
+ assert outcome["ok"] == [_STATUS["slug"]], outcome
+ posts = [c for c in calls if "/statuses/" in c[1]]
+ assert posts == [posts[0]] and posts[0][1] == f"repos/unslothai/unsloth/statuses/{full}"
+ assert not any(c[1].endswith("/statuses/2ecb19df") for c in calls)
+ # Every field is its own argument: nothing is assembled into a shell line.
+ assert "-f" in posts[0] and f"description={_STATUS['description']}" in posts[0]
+
+
+def test_a_commit_that_no_longer_exists_is_reported_not_silently_dropped(monkeypatch, capsys):
+ """A force-push can remove the commit a running kernel was dispatched for.
+ Posting is then impossible, but it must SAY so and release the kernel rather
+ than retry forever."""
+ calls = _fake_gh(monkeypatch, resolve_to = None)
+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")
+ assert outcome["unresolved"] == [_STATUS["slug"]] and outcome["failed"] == []
+ assert not any("/statuses/" in c[1] for c in calls), "posted to a commit that is gone"
+ assert "Could not resolve a collected commit" in capsys.readouterr().out
+
+
+def test_every_workflow_posts_through_the_shared_poster_and_none_keeps_a_shell_loop():
+ for path in (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF):
+ body = path.read_text(encoding = "utf-8")
+ assert "kaggle_t4_ci/post_statuses.py" in body, f"{path.name} does not use the poster"
+ assert "statuses.txt" not in body, f"{path.name} still carries the tab-delimited loop"
+
+
+# ---------------------------------------------- post, then delete; never the reverse
+
+
+ "path", (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF), ids = ("notebook", "studio", "collect")
+)
+def test_collection_never_deletes_and_the_release_step_comes_after_posting(path):
+ """A verdict that does not reach GitHub must leave its kernel up for the
+ next pass: collect --no-delete, post, then --delete-collected releases only
+ what was delivered."""
+ body = path.read_text(encoding = "utf-8")
+ steps = [(name, step) for _job, name, step in _steps(_wf(path))]
+ collect_steps = [
+ s
+ for _n, s in steps
+ if "kaggle_t4_ci/collect.py" in (s.get("run") or "")
+ and "--delete-collected" not in s["run"]
+ ]
+ assert collect_steps, f"{path.name} never collects"
+ for step in collect_steps:
+ assert (
+ "--no-delete" in step["run"]
+ ), f"{path.name} collects with deletion on:\n{step['run']}"
+ names = [n for n, _s in steps]
+ post = next(i for i, (n, s) in enumerate(steps) if "post_statuses.py" in (s.get("run") or ""))
+ release = next(
+ i for i, (n, s) in enumerate(steps) if "--delete-collected" in (s.get("run") or "")
+ )
+ assert (
+ release > post
+ ), f"{path.name} releases kernels before posting: {names[release]!r} before {names[post]!r}"
+ assert "--posted kaggle_collected/posted.json" in steps[release][1]["run"]
+
+
+def test_a_kernel_whose_status_did_not_post_is_KEPT_for_the_next_pass(tmp_path, monkeypatch):
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ result = tmp_path / "collect_result.json"
+ result.write_text(
+ json.dumps(
+ {
+ "kernels": [
+ {"slug": "me/ok", "verdict": "pass", "deleted": False},
+ {"slug": "me/refused", "verdict": "fail", "deleted": False},
+ {"slug": "me/legacy", "verdict": "reaped", "deleted": False},
+ {"slug": "me/running", "verdict": "pending", "deleted": False},
+ ],
+ "statuses": [],
+ }
+ )
+ )
+ posted = tmp_path / "posted.json"
+ posted.write_text(json.dumps({"ok": ["me/ok"], "failed": ["me/refused"], "unresolved": []}))
+ assert collect.delete_collected(result, posted) == 0
+ assert deleted == ["me/ok", "me/legacy"], deleted
+ outcome = json.loads((tmp_path / "delete_result.json").read_text())
+ assert outcome["kept"] == ["me/refused"]
+
+
+def test_no_delivery_record_at_all_keeps_every_kernel_that_had_something_to_post(
+ tmp_path, monkeypatch
+):
+ """The poster never ran, so deleting would lose every verdict of the pass."""
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ result = tmp_path / "collect_result.json"
+ result.write_text(
+ json.dumps(
+ {
+ "kernels": [
+ {"slug": "me/a", "verdict": "pass", "deleted": False},
+ {"slug": "me/nothing-to-post", "verdict": "infra", "deleted": False},
+ ],
+ "statuses": [{"slug": "me/a", "slugs": ["me/a"]}],
+ }
+ )
+ )
+ collect.delete_collected(result, tmp_path / "posted.json")
+ assert deleted == ["me/nothing-to-post"], deleted
+
+
+def test_a_refused_post_is_red_and_recorded(monkeypatch, tmp_path):
+ _fake_gh(monkeypatch, resolve_to = "2ecb19df" + "b" * 32, post_ok = False)
+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")
+ assert outcome["failed"] == [_STATUS["slug"]]
+ result = tmp_path / "r.json"
+ result.write_text(json.dumps({"statuses": [dict(_STATUS)]}))
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "post_statuses",
+ "--result",
+ str(result),
+ "--out",
+ str(tmp_path / "p.json"),
+ "--repo",
+ "o/r",
+ ],
+ )
+ assert post_statuses.main() == 1
+ assert json.loads((tmp_path / "p.json").read_text())["failed"] == [_STATUS["slug"]]
+
+
+def test_the_poster_signs_only_records_it_recognises(monkeypatch, capsys):
+ calls = _fake_gh(monkeypatch, resolve_to = "2ecb19df" + "c" * 32)
+ bad_state = dict(_STATUS, state = "green")
+ bad_context = dict(_STATUS, context = "ci/somebody-else")
+ outcome = post_statuses.post_all([bad_state, bad_context], "o/r")
+ assert outcome["invalid"] == [_STATUS["slug"], _STATUS["slug"]]
+ assert calls == [], "a malformed record reached gh"
+
+
+# ------------------------------------------------- what a status is built from
+
+
+def test_two_kernels_for_one_commit_and_context_post_ONE_status_and_a_failure_wins():
+ """Slots 1 and 2 on one sha are two kernels under one context, and two
+ statuses would race, the last posted becoming the visible verdict."""
+ records = [
+ {
+ "slug": "me/one",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "verdict": "pass",
+ "reason": "all passed",
+ },
+ {
+ "slug": "me/two",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "verdict": "fail",
+ "reason": "1 of 5 failed",
+ },
+ {
+ "slug": "me/other",
+ "sha": "abcdef02",
+ "kind": "notebook",
+ "verdict": "pass",
+ "reason": "all passed",
+ },
+ ]
+ statuses = collect.statuses_from(records)
+ assert len(statuses) == 2
+ joint = next(s for s in statuses if s["sha"] == "abcdef01")
+ assert joint["state"] == "failure" and sorted(joint["slugs"]) == ["me/one", "me/two"]
+ # Order must not matter: the failure wins whichever kernel was listed first.
+ statuses = collect.statuses_from(list(reversed(records)))
+ assert next(s for s in statuses if s["sha"] == "abcdef01")["state"] == "failure"
+
+
+def test_a_description_is_one_line():
+ record = {
+ "slug": "me/k",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "verdict": "fail",
+ "reason": "a\nb\tc",
+ }
+ assert collect.statuses_from([record])[0]["description"] == "fail: a b c"
+
+
+# ------------------------------------ the evidence decides, and only complete evidence
+
+
+def _terminal_entry():
+ return {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 0.4,
+ }
+
+
+def test_an_incomplete_download_judges_nothing_and_keeps_the_kernel(tmp_path, monkeypatch):
+ """fetch_evidence flags a spent budget instead of raising. Judging the short
+ set reads a run that lost half its notebooks as whatever survived, and
+ deleting makes that permanent."""
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ launch,
+ "fetch_evidence",
+ lambda slug, dest, deadline = None: {"notebooks": ["a"], "truncated": True},
+ )
+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})
+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "pending", record
+ assert deleted == []
+ assert collect.statuses_from([record]) == []
+
+
+def test_a_kernel_another_collector_finished_first_posts_nothing(tmp_path, monkeypatch):
+ """Three unserialised workflows collect on one account, and the loser of
+ that race must not post an `infra` over the winner's real verdict."""
+
+ def _gone(
+ slug,
+ dest,
+ deadline = None,
+ ):
+ raise RuntimeError(
+ "404 Client Error: Not Found for url: https://www.kaggle.com/api/v1/kernels/output"
+ )
+
+ deleted: list[str] = []
+ monkeypatch.setattr(launch, "fetch_evidence", _gone)
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})
+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "gone", record
+ assert deleted == [] and collect.statuses_from([record]) == []
+
+
+def test_an_unreadable_report_is_infra_not_a_crash(tmp_path, monkeypatch):
+ """Raising here writes no result and wedges every later pass. Nothing was
+ learned, so `infra`, and the kernel is released."""
+
+ def _boom(dest):
+ raise AttributeError("'list' object has no attribute 'get'")
+
+ monkeypatch.setattr(
+ launch,
+ "fetch_evidence",
+ lambda slug, dest, deadline = None: {"notebooks": ["a"], "truncated": False},
+ )
+ monkeypatch.setattr(launch, "extract_reports", _boom)
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})
+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "infra" and "could not be read" in record["reason"]
+ assert record["deleted"] is True
+
+
+def test_a_malformed_report_line_is_skipped_by_the_extractor(tmp_path):
+ (tmp_path / "kernel.log").write_text(
+ 'T4_SMOKE_REPORT []\nT4_SMOKE_REPORT "text"\nT4_SMOKE_REPORT {"label": "ok", "passed": true}\n',
+ encoding = "utf-8",
+ )
+ assert launch.extract_reports(tmp_path) == [{"label": "ok", "passed": True}]
+
+
+def test_the_expected_report_count_travels_inside_the_kernel(tmp_path, monkeypatch):
+ """The scheduled collector has no workflow output to read a payload count
+ from, and judging a five-payload kernel against one turns a run that lost
+ four into a pass. The driver's own sentinel is the record."""
+
+ def _fetch(
+ slug,
+ dest,
+ deadline = None,
+ ):
+ dest.mkdir(parents = True, exist_ok = True)
+ (dest / "kernel.log").write_text(
+ 'KAGGLE_T4_CI_DRIVER_EXPECT {"reports": 5}\nT4_SMOKE_REPORT {"label": "a", "passed": true}\n',
+ encoding = "utf-8",
+ )
+ return {"notebooks": [], "log": "kernel.log", "truncated": False}
+
+ monkeypatch.setattr(launch, "fetch_evidence", _fetch)
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)
+ api = _StubApi([], {"me/unsloth-t4-ci-nabcdef01-1111": "COMPLETE"})
+ record = collect.collect_one(api, _terminal_entry(), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["expected"] == 5
+ assert record["verdict"] == "partial", record
+ # And with no sentinel the caller's number stands.
+ (tmp_path / "plain").mkdir()
+ assert collect.expected_reports(tmp_path / "plain", 3) == 3
+
+
+def test_the_built_driver_records_its_own_expected_report_count():
+ sys.path.insert(0, str(CI_DIR))
+ import build_kernel
+
+ payloads = {
+ "t4_a.ipynb": {"cells": []},
+ "t4_b.ipynb": {"cells": []},
+ "studio_install.ipynb": {"cells": []},
+ }
+
+ def _source(nb):
+ return "".join("".join(cell.get("source", [])) for cell in nb["cells"])
+
+ driver = _source(build_kernel.build_driver(payloads, 60, cpu_lane = "studio_install.ipynb"))
+ assert 'KAGGLE_T4_CI_DRIVER_EXPECT " + json.dumps({"reports": 2})' in driver
+ plain = _source(build_kernel.build_driver({"t4_a.ipynb": {"cells": []}}, 60))
+ assert 'json.dumps({"reports": 1})' in plain
+
+
+def test_the_scheduled_collector_no_longer_guesses_the_payload_count():
+ body = COLLECT_WF.read_text(encoding = "utf-8")
+ assert (
+ "--expect" not in body
+ ), "the scheduled collector still applies one flat --expect to every kernel"
+
+
+# ------------------------------------------------------------- the reaper's reach
+
+
+def test_a_kernel_far_past_the_ceiling_is_still_seen(monkeypatch):
+ """A pass delayed six hours by an outage still finds the wedge it exists
+ for; ageing it out of the listing leaves it billing with its commit
+ pending forever."""
+ from datetime import datetime, timedelta
+
+ now = datetime(2026, 9, 6, 12, 0, 0)
+ old = _StubKernel("me/unsloth-t4-ci-nabcdef01-1111", last_run_time = now - timedelta(hours = 30))
+ api = _StubApi([old], {})
+ found = collect.find_ours(api, now = now, max_age_hours = 3.0)
+ assert [f["slug"] for f in found] == ["me/unsloth-t4-ci-nabcdef01-1111"]
+ assert found[0]["age_hours"] == pytest.approx(30.0)
+
+
+class _PagedApi(_StubApi):
+ """kernels_list answering from a list of pages."""
+
+ def __init__(self, pages, statuses):
+ super().__init__([], statuses)
+ self._pages = pages
+ self.pages_asked: list[int] = []
+
+ def kernels_list(
+ self,
+ mine = True,
+ page = 1,
+ page_size = 100,
+ sort_by = "dateRun",
+ ):
+ self.pages_asked.append(page)
+ return self._pages[page - 1] if page <= len(self._pages) else []
+
+
+def test_a_kernel_of_ours_below_five_hundred_newer_records_is_still_reached():
+ """The account is shared and human kernels are never deleted, so a fixed
+ page cap would leave a kernel of ours below it unreachable forever. The walk
+ stops only at a page entirely past the horizon with none of ours on it."""
+ from datetime import datetime, timedelta
+
+ now = datetime(2026, 9, 6, 12, 0, 0)
+ recent = now - timedelta(hours = 1)
+ human = [
+ [_StubKernel(f"me/human-notebook-{p}-{i}", last_run_time = recent) for i in range(100)]
+ for p in range(6)
+ ]
+ ours = _StubKernel(
+ "me/unsloth-t4-ci-nabcdef012345-1111", last_run_time = now - timedelta(hours = 5)
+ )
+ pages = human + [[ours] + [_StubKernel(f"me/old-{i}", last_run_time = recent) for i in range(99)]]
+ api = _PagedApi(pages, {})
+ found = collect.find_ours(api, now = now)
+ assert [f["slug"] for f in found] == [ours.ref]
+ assert 7 in api.pages_asked
+
+
+def test_the_listing_walk_ends_at_a_page_past_the_horizon_with_none_of_ours():
+ from datetime import datetime, timedelta
+
+ now = datetime(2026, 9, 6, 12, 0, 0)
+ ancient = now - timedelta(hours = collect.LISTING_HORIZON_HOURS + 1)
+ pages = [
+ [_StubKernel(f"me/human-{i}", last_run_time = now - timedelta(hours = 1)) for i in range(100)],
+ [_StubKernel(f"me/ancient-{i}", last_run_time = ancient) for i in range(100)],
+ [_StubKernel("me/unsloth-t4-ci-nabcdef012345-2222", last_run_time = ancient)],
+ ]
+ api = _PagedApi(pages, {})
+ assert collect.find_ours(api, now = now) == []
+ assert api.pages_asked == [1, 2], "the walk did not stop at the first page past the horizon"
+ # A page past the horizon that still carries one of ours keeps the walk going.
+ pages[1][50] = _StubKernel("me/unsloth-t4-ci-nabcdef012345-3333", last_run_time = ancient)
+ api = _PagedApi(pages, {})
+ found = collect.find_ours(api, now = now)
+ assert {f["slug"] for f in found} == {
+ "me/unsloth-t4-ci-nabcdef012345-3333",
+ "me/unsloth-t4-ci-nabcdef012345-2222",
+ }
+
+
+def test_the_listing_walk_stops_at_the_pass_deadline(monkeypatch):
+ from datetime import datetime, timedelta
+
+ now = datetime(2026, 9, 6, 12, 0, 0)
+ pages = [
+ [
+ _StubKernel(f"me/human-{p}-{i}", last_run_time = now - timedelta(hours = 1))
+ for i in range(100)
+ ]
+ for p in range(5)
+ ]
+ api = _PagedApi(pages, {})
+ clock = [1000.0]
+ monkeypatch.setattr(collect.time, "time", lambda: clock[0])
+ orig = api.kernels_list
+
+ def slow(*a, **k):
+ clock[0] += 400.0
+ return orig(*a, **k)
+
+ api.kernels_list = slow
+ collect.find_ours(api, now = now, deadline = 1500.0)
+ assert api.pages_asked == [1, 2], api.pages_asked
+ source = (CI_DIR / "collect.py").read_text(encoding = "utf-8")
+ assert "find_ours(api, max_age_hours = args.max_age_hours, deadline = deadline)" in source
+
+
+def test_a_reaped_kernel_is_not_reported_deleted_before_it_is(tmp_path, monkeypatch):
+ """The delete happens in a later step that can be refused, and this reason
+ is posted as a durable status before it runs, so it must not claim it."""
+ from datetime import datetime, timedelta
+
+ slug = "me/unsloth-t4-ci-nabcdef012345-1111"
+ api = _StubApi([], {slug: "RUNNING"})
+ entry = {
+ "slug": slug,
+ "sha": "abcdef012345",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 5.0,
+ }
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0, delete = False)
+ assert record["verdict"] == "reaped"
+ assert "released for deletion" in record["reason"] and "was deleted" not in record["reason"]
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0, delete = True)
+ assert "was deleted" in record["reason"]
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: False)
+ record = collect.collect_one(api, entry, tmp_path, expect = 1, max_age_hours = 3.0, delete = True)
+ assert "was deleted" not in record["reason"] and record["deleted"] is False
+
+
+def test_a_malformed_neighbour_notebook_does_not_hide_a_failing_report(tmp_path, monkeypatch):
+ """One malformed notebook among several used to make the reader raise, and
+ the collector then judged the kernel infra, posted success and released it.
+ The failing report in the well-formed neighbour must survive."""
+ slug = _ENTRY["slug"]
+ dest = tmp_path / slug.rsplit("/", 1)[-1]
+ dest.mkdir()
+ failing = launch.RESULT_PREFIX + json.dumps({"label": "control", "passed": False, "model": "m"})
+ (dest / f"a{launch.OUTPUT_SUFFIX}").write_text(
+ json.dumps({"cells": [{"outputs": [{"text": failing + "\n"}]}]}), encoding = "utf-8"
+ )
+ (dest / f"b{launch.OUTPUT_SUFFIX}").write_text("[]", encoding = "utf-8")
+ (dest / f"c{launch.OUTPUT_SUFFIX}").write_text(
+ json.dumps(
+ {
+ "cells": [
+ "not a cell",
+ {"outputs": ["not an output", {"text": 7}, {"text": ["x", 1]}]},
+ ]
+ }
+ ),
+ encoding = "utf-8",
+ )
+ assert launch.extract_reports(dest) == [{"label": "control", "passed": False, "model": "m"}]
+ monkeypatch.setattr(launch, "fetch_evidence", lambda slug, dest, deadline = None: {})
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)
+ api = _StubApi([], {slug: "COMPLETE"})
+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "fail", record
+ assert "report_error" not in record
+
+
+# ------------------------------------------------------- who is allowed to be quiet
+
+
+def test_the_scheduled_collector_fails_loudly_when_it_cannot_authenticate(
+ tmp_path, monkeypatch, capsys
+):
+ """Its token is the repository's own, so a green pass collecting nothing
+ hides an expired credential while kernels bill and commits stay pending."""
+
+ def _no(*a, **k):
+ raise OSError("Could not find kaggle.json")
+
+ monkeypatch.setattr(launch, "_api", _no)
+ monkeypatch.setenv("KAGGLE_API_TOKEN", "a-token-kaggle-refused")
+ monkeypatch.setattr(sys, "argv", ["collect", "--outdir", str(tmp_path), "--require-auth"])
+ assert collect.main() == 1
+ assert "Kaggle authentication failed" in capsys.readouterr().out
+ monkeypatch.setattr(sys, "argv", ["collect", "--outdir", str(tmp_path)])
+ assert collect.main() == 0, "a fork job with the secret withheld is still a skip"
+
+
+def test_an_unconfigured_collector_account_warns_and_passes(tmp_path, monkeypatch, capsys):
+ """The matrix is static, so a repository with one Kaggle account leaves the
+ second secret unset; that leg used to go red every ten minutes for an
+ account that does not exist. An empty token is an absent account, warned
+ about and green; a token present and refused is still red."""
+
+ def _no(*a, **k):
+ raise OSError("Could not find kaggle.json")
+
+ monkeypatch.setattr(launch, "_api", _no)
+ monkeypatch.delenv("KAGGLE_API_TOKEN", raising = False)
+ monkeypatch.setattr(sys, "argv", ["collect", "--outdir", str(tmp_path), "--require-auth"])
+ assert collect.main() == 0
+ out = capsys.readouterr().out
+ assert "::warning title=Kaggle account not configured" in out
+ assert "::error" not in out
+ monkeypatch.setenv("KAGGLE_API_TOKEN", "")
+ assert collect.main() == 0
+ monkeypatch.setenv("KAGGLE_API_TOKEN", "present")
+ assert collect.main() == 1
+ body = COLLECT_WF.read_text(encoding = "utf-8")
+ assert "EMPTY token" in body
+
+
+def test_the_scheduled_workflow_asks_for_that():
+ body = COLLECT_WF.read_text(encoding = "utf-8")
+ assert "--require-auth" in body
+
+
+def test_the_collector_installs_the_client_the_gpu_workflows_install():
+ """1.7.4.5 cannot read KAGGLE_API_TOKEN at all (#9535), so a scheduled pass
+ on it authenticated nothing and, before --require-auth, said so to nobody."""
+ pins = {}
+ for path in (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF):
+ found = re.findall(
+ r"pip install [^\n]*'kaggle==([0-9][^']*)'", path.read_text(encoding = "utf-8")
+ )
+ assert found, f"{path.name} pins no kaggle client"
+ pins[path.name] = set(found)
+ assert (
+ len(set().union(*pins.values())) == 1
+ ), f"the workflows disagree on the kaggle client: {pins}"
+
+
+# -------------------------------------------------- what a dispatch is allowed to be
+
+
+def test_dispatch_refuses_a_ref_that_is_not_a_commit(monkeypatch, capsys):
+ """`slug_name` falls back to the legacy unattributable form for anything not
+ hex, so a branch here pushes a kernel that runs, bills and reports to
+ nobody."""
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "launch",
+ "--notebook",
+ "k.ipynb",
+ "--outdir",
+ "out",
+ "--dispatch",
+ "--kind",
+ "studio",
+ "--commit-sha",
+ "main",
+ ],
+ )
+ with pytest.raises(SystemExit) as exc:
+ launch.main()
+ assert exc.value.code == 2
+ assert "hex commit id" in capsys.readouterr().err
+
+
+def test_the_studio_workflow_resolves_its_ref_to_a_commit_before_dispatching():
+ """workflow_dispatch accepts any `unsloth_ref`; the notebook workflow
+ already resolved it, the Studio one passed it straight through."""
+ for path in (NOTEBOOK_WF, STUDIO_WF):
+ wf = _wf(path)
+ gate_ref = next(s for s in wf["jobs"]["gate"]["steps"] if s.get("id") == "ref")
+ assert (
+ "git ls-remote" in gate_ref["run"] and "[0-9a-f]{40}" in gate_ref["run"]
+ ), f"{path.name}: the gate does not resolve refs"
+ step = next(s for _j, n, s in _steps(wf) if n == "Resolve the ref under test")
+ run = step["run"]
+ # Resolved ONCE, by the gate, and reused: a branch re-resolved after
+ # the job queued can name a commit the gate never keyed anything on.
+ assert step["env"]["GATE_SHA"] == "${{ needs.gate.outputs.head_sha }}"
+ assert (
+ 'RESOLVED="$GATE_SHA"' in run and "$(git ls-remote" not in run
+ ), f"{path.name}: the GPU job resolves the ref a second time"
+ assert wf["jobs"]["gate"]["outputs"]["head_sha"] == "${{ steps.ref.outputs.head_sha }}"
+ assert (
+ "exit 1" in run or "stand_down=true" in run
+ ), f"{path.name} dispatches on an unresolved ref"
+ # A full SHA passes the shape test whether or not the repository has it;
+ # only fetching the object says it is there. Without this the Studio leg
+ # spent a session on a commit no status could be posted to.
+ assert (
+ "git fetch --quiet --depth=1 https://github.com/unslothai/unsloth" in run
+ ), f"{path.name} dispatches a full SHA without checking the repository serves it"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_pending_is_posted_only_for_a_kernel_that_was_actually_dispatched(path):
+ """launch.py exits 0 on every stand-down, and a pending status for a kernel
+ that does not exist is one no collector can ever replace."""
+ step = next(
+ s for _j, n, s in _steps(_wf(path)) if n == "Mark the dispatch pending on this commit"
+ )
+ assert step.get("if") == "steps.launch.outputs.verdict == 'dispatched'", step.get("if")
+
+
+def test_a_lookup_that_could_not_be_made_keeps_the_kernel(monkeypatch, capsys):
+ """Only GitHub saying the commit is not there releases a kernel: a 5xx or a
+ dropped connection says nothing, and reading it as "gone" deletes the only
+ copy of the result."""
+ calls = _fake_gh(monkeypatch, lookup_error = "gh: HTTP 503 Service Unavailable")
+ outcome = post_statuses.post_all([dict(_STATUS)], "unslothai/unsloth")
+ assert outcome["failed"] == [_STATUS["slug"]]
+ assert outcome["unresolved"] == []
+ assert not any("/statuses/" in c[1] for c in calls)
+ assert "Commit lookup failed" in capsys.readouterr().out
+
+
+def test_resolve_sha_separates_missing_from_unknown(monkeypatch):
+ """Measured: a commit that is not there answers HTTP 422 "No commit found
+ for SHA", not a 404. Anything else is an unknown, never a missing."""
+ answers = {}
+
+ def _gh(args):
+ return answers[args[1]]
+
+ monkeypatch.setattr(post_statuses, "_gh", _gh)
+ answers["repos/o/r/commits/aa"] = (0, "a" * 40, "")
+ answers["repos/o/r/commits/bb"] = (1, "", "gh: No commit found for SHA: bb (HTTP 422)")
+ answers["repos/o/r/commits/cc"] = (1, "", "gh: HTTP 502 Bad Gateway")
+ answers["repos/o/r/commits/dd"] = (1, "", "")
+ # The status code alone is not the answer: 404 is also an unreadable
+ # repository, 422 also an ambiguous abbreviation.
+ answers["repos/o/r/commits/ee"] = (1, "", "gh: Not Found (HTTP 404)")
+ answers["repos/o/r/commits/ff"] = (1, "", "gh: Validation Failed (HTTP 422)")
+ assert post_statuses.resolve_sha("o/r", "aa") == ("ok", "a" * 40)
+ assert post_statuses.resolve_sha("o/r", "bb") == ("missing", None)
+ for sha in ("cc", "dd", "ee", "ff"):
+ assert post_statuses.resolve_sha("o/r", sha) == ("error", None), sha
+
+
+# ------------------------------------------ the terminal path is for known states only
+
+
+_ENTRY = {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "legacy": False,
+ "age_hours": 0.4,
+}
+
+
+def test_an_unrecognised_kernel_state_is_kept_and_not_judged(tmp_path, monkeypatch):
+ """Kaggle's enum can grow, and a non-terminal state has no evidence: judging
+ one posts a green `infra` for a run still to come, then deletes it."""
+ touched: list[str] = []
+ monkeypatch.setattr(
+ launch, "fetch_evidence", lambda slug, dest, deadline = None: touched.append(slug) or {}
+ )
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: touched.append("delete") or True
+ )
+ api = _StubApi([], {_ENTRY["slug"]: "NEW_SCRIPT"})
+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "pending"
+ assert record["verdict"] not in collect.DELETABLE
+ assert collect.statuses_from([record]) == []
+ assert touched == [], "an unjudged kernel must be neither downloaded nor deleted"
+
+
+def test_a_downloaded_file_that_is_not_a_notebook_does_not_wedge_the_collector(
+ tmp_path, monkeypatch
+):
+ """`[]` is valid JSON and not a notebook. The expected-count reader used to
+ raise on it, and a collector that raises writes no result and stalls on the
+ same kernel every pass."""
+ slug = _ENTRY["slug"]
+ dest = tmp_path / slug.rsplit("/", 1)[-1]
+ dest.mkdir()
+ (dest / f"x{launch.OUTPUT_SUFFIX}").write_text("[]", encoding = "utf-8")
+ (dest / f"y{launch.OUTPUT_SUFFIX}").write_text(
+ json.dumps({"cells": ["not a cell", {"outputs": ["not an output", {"text": 7}]}]}),
+ encoding = "utf-8",
+ )
+ monkeypatch.setattr(launch, "fetch_evidence", lambda slug, dest, deadline = None: {})
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)
+ api = _StubApi([], {slug: "COMPLETE"})
+ assert collect.expected_reports(dest, 4) == 4
+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 4, max_age_hours = 3.0)
+ assert record["verdict"] == "infra", record
+ # A notebook kernel with no expected-count record has an UNKNOWN plan.
+ assert record["expected"] is None
+
+
+def test_the_evidence_download_is_clamped_to_the_pass_deadline(tmp_path, monkeypatch):
+ """A kernel started just inside the pass budget must not get a fresh
+ five-minute download budget, or the collector overruns the job with the
+ release step still to come."""
+ seen: list[float] = []
+
+ def _fetch(
+ slug,
+ dest,
+ deadline = None,
+ ):
+ seen.append(deadline)
+ return {}
+
+ monkeypatch.setattr(launch, "fetch_evidence", _fetch)
+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)
+ api = _StubApi([], {_ENTRY["slug"]: "COMPLETE"})
+ pass_deadline = time.time() + 5.0
+ collect.collect_one(
+ api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0, deadline = pass_deadline
+ )
+ assert seen and seen[0] <= pass_deadline
+ # And the collector's loop hands its deadline down at the one call site.
+ source = (CI_DIR / "collect.py").read_text(encoding = "utf-8")
+ body = source[source.index("deadline = time.time() + BUDGET_SEC") :]
+ assert "deadline = deadline," in body[: body.index("statuses_from")]
+
+
+# ---------------------------------------- collect BEFORE the recheck, never between
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_collection_runs_before_the_recheck_so_the_recheck_is_last_before_the_push(path):
+ """Collection can spend up to BUDGET_SEC downloading, and between the
+ recheck and the push that gap is the stale window the recheck exists to
+ close: the other GPU workflow can take the last session in it."""
+ steps = [(n, s) for _j, n, s in _steps(_wf(path))]
+ names = [n for n, _s in steps]
+ collect_i = names.index("Collect finished Kaggle runs")
+ recheck_i = names.index("Recheck the Kaggle account")
+ launch_i = names.index("Dispatch to Kaggle")
+ assert collect_i < recheck_i < launch_i, names[collect_i : launch_i + 1]
+ assert "steps.recheck" not in (steps[collect_i][1].get("if") or "")
+ # Nothing that takes BUDGET_SEC sits between the recheck and the push.
+ between = [n for n in names[recheck_i + 1 : launch_i]]
+ assert not any("collect.py" in (s.get("run") or "") for n, s in steps if n in between), between
+ assert "steps.recheck.outputs.should_run == 'true'" in steps[launch_i][1]["if"]
+
+
+# ------------------------------------- collected Studio evidence is unpacked too
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_the_unpack_step_reads_the_tree_a_collected_kernel_lands_in(path):
+ """A collected kernel lands in kaggle_collected/<slug>, so an unpack step
+ reading only kaggle_evidence never fires on the run that retrieved the
+ screenshots."""
+ step = next(s for _j, n, s in _steps(_wf(path)) if n == "Unpack the Playwright evidence")
+ assert "kaggle_collected/**/*_output.ipynb" in step["if"]
+ assert "kaggle_evidence/**/*_output.ipynb" in step["if"]
+ run = step["run"]
+ assert "kaggle_collected/*/" in run and "kaggle_evidence/*/" in run
+ # One kernel per call: chunks are numbered per bundle, so a walk over two
+ # would splice them together.
+ assert '--evidence "$dir"' in run and 'studio_evidence/$(basename "$dir")' in run
+
+
+def test_the_scheduled_collector_unpacks_and_uploads_what_it_collected():
+ """It collects most Studio kernels, so it is the pass that has to put the
+ screenshots back and keep them somewhere."""
+ steps = [(n, s) for _j, n, s in _steps(_wf(COLLECT_WF))]
+ names = [n for n, _s in steps]
+ unpack = steps[names.index("Unpack the Playwright evidence")][1]
+ assert "kaggle_collected/*/" in unpack["run"] and "collect_evidence.py" in unpack["run"]
+ upload = steps[names.index("Upload evidence")][1]
+ assert upload["uses"].startswith("actions/upload-artifact@")
+ assert "kaggle_collected/**" in upload["with"]["path"]
+ assert "studio_evidence/**" in upload["with"]["path"]
+ assert upload.get("continue-on-error") is True, "an artifact outage is not a collection failure"
+ assert names.index("Upload evidence") > names.index("Post the commit statuses")
+
+
+def test_a_listed_notebook_without_a_download_url_marks_the_evidence_incomplete(
+ tmp_path, monkeypatch
+):
+ """Skipping it silently left `truncated` false, so the short set was judged,
+ posted and its kernel deleted: a failing payload lost for good."""
+ listing = {
+ "files": [{"fileName": f"a{launch.OUTPUT_SUFFIX}"}],
+ "log": "",
+ "truncated": False,
+ }
+ monkeypatch.setattr(launch, "list_outputs", lambda slug, timeout = None, deadline = None: listing)
+ evidence = launch.fetch_evidence("me/k", tmp_path / "k", deadline = time.time() + 60)
+ assert evidence["notebooks"] == []
+ assert evidence["truncated"] is True
+
+
+def test_old_and_new_slug_forms_for_one_commit_post_one_status(monkeypatch):
+ """An 8 and a 12 character slug can name one commit in one pass, and only
+ the commits API can say so: the collector keeps them apart and the POSTER
+ merges what resolves alike, failure winning, both slugs named."""
+ full = "abcdef012345" + "0" * 28
+ calls: list[list[str]] = []
+
+ def _gh(args):
+ calls.append(list(args))
+ if "/commits/" in args[1]:
+ return (0, full, "")
+ return (0, "", "")
+
+ monkeypatch.setattr(post_statuses, "_gh", _gh)
+ records = [
+ {
+ "slug": "me/unsloth-t4-ci-nabcdef012345-2222",
+ "sha": "abcdef012345",
+ "kind": "notebook",
+ "verdict": "fail",
+ "reason": "leg red",
+ },
+ {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "verdict": "pass",
+ "reason": "ok",
+ },
+ ]
+ statuses = collect.statuses_from(records)
+ assert len(statuses) == 2, "the collector cannot know these are one commit"
+ for order in (statuses, statuses[::-1]):
+ calls.clear()
+ outcome = post_statuses.post_all([dict(s) for s in order], "unslothai/unsloth")
+ posts = [c for c in calls if "/statuses/" in c[1]]
+ assert len(posts) == 1, posts
+ assert "state=failure" in " ".join(posts[0])
+ assert sorted(outcome["ok"]) == sorted(r["slug"] for r in records)
+
+
+def test_two_commits_that_merely_share_eight_characters_post_two_statuses():
+ """The reason the slug grew to twelve characters: merged, one commit loses
+ its verdict and has its kernel deleted as delivered."""
+ records = [
+ {
+ "slug": "me/unsloth-t4-ci-nabcdef012345-1111",
+ "sha": "abcdef012345",
+ "kind": "notebook",
+ "verdict": "pass",
+ "reason": "ok",
+ },
+ {
+ "slug": "me/unsloth-t4-ci-nabcdef01ffff-2222",
+ "sha": "abcdef01ffff",
+ "kind": "notebook",
+ "verdict": "fail",
+ "reason": "leg red",
+ },
+ ]
+ statuses = collect.statuses_from(records)
+ assert len(statuses) == 2, statuses
+ assert {s["sha"]: s["state"] for s in statuses} == {
+ "abcdef012345": "success",
+ "abcdef01ffff": "failure",
+ }
+ assert all(len(s["slugs"]) == 1 for s in statuses)
+
+
+def test_the_release_phase_stops_at_its_budget_and_keeps_the_rest(tmp_path, monkeypatch):
+ """delete_kernel allows three 180-second attempts per kernel, so an
+ unbounded release could outlive the job and be killed mid-delete. What is
+ left over is released by the next pass."""
+ clock = [1000.0]
+ monkeypatch.setattr(collect.time, "time", lambda: clock[0])
+
+ def _slow_delete(slug, deadline = None):
+ clock[0] += collect.RELEASE_BUDGET_SEC # one delete eats the whole budget
+ return True
+
+ monkeypatch.setattr(launch, "delete_kernel", _slow_delete)
+ kernels = [
+ {"slug": f"me/unsloth-t4-ci-nabcdef01234{i}-1111", "verdict": "pass"} for i in range(3)
+ ]
+ result = tmp_path / "collect_result.json"
+ result.write_text(json.dumps({"kernels": kernels, "statuses": []}), encoding = "utf-8")
+ posted = tmp_path / "posted.json"
+ posted.write_text(json.dumps({"ok": [], "failed": []}), encoding = "utf-8")
+ assert collect.delete_collected(result, posted) == 0
+ outcome = json.loads((tmp_path / "delete_result.json").read_text())
+ assert len(outcome["deleted"]) == 1 and len(outcome["kept"]) == 2, outcome
+
+
+def test_the_scheduled_collector_timeout_covers_collection_and_release():
+ job = _wf(COLLECT_WF)["jobs"]["collect"]
+ floor = (collect.BUDGET_SEC + collect.RELEASE_BUDGET_SEC) / 60 + 3
+ assert job["timeout-minutes"] >= floor, (job["timeout-minutes"], floor)
+
+
+def test_nothing_tells_the_reader_to_require_a_sampled_context():
+ """The gate samples and the workflows path-filter, so a commit the gate
+ skips never gets a status, and a required context that never arrives blocks
+ the merge."""
+ for path in (NOTEBOOK_WF, STUDIO_WF, COLLECT_WF, CI_DIR / "launch.py", CI_DIR / "collect.py"):
+ assert "branch protection must require" not in path.read_text(encoding = "utf-8"), path
+
+
+def test_a_delete_is_clamped_to_the_deadline_it_is_handed(monkeypatch):
+ """Checked only before the call, the deadline did not bound the release:
+ two deletes of three 180-second attempts outlive a 600-second budget. Every
+ attempt is clamped to what is left and none starts past it."""
+ clock = [1000.0]
+ timeouts: list[float] = []
+
+ class _Proc:
+ returncode = 1
+ stdout = ""
+ stderr = "503"
+
+ def _run(
+ cmd,
+ capture_output = True,
+ text = True,
+ timeout = None,
+ ):
+ timeouts.append(timeout)
+ clock[0] += timeout # the call uses its whole allowance
+ return _Proc()
+
+ monkeypatch.setattr(launch.subprocess, "run", _run)
+ monkeypatch.setattr(launch.time, "time", lambda: clock[0])
+ monkeypatch.setattr(launch.time, "sleep", lambda s: clock.__setitem__(0, clock[0] + s))
+ assert launch.delete_kernel("me/k", deadline = 1000.0 + 200.0) is False
+ assert timeouts and timeouts[0] <= 200.0
+ assert sum(timeouts) <= 200.0, timeouts
+ assert len(timeouts) < launch.DELETE_ATTEMPTS, "an attempt started past the deadline"
+
+
+def test_the_release_hands_every_delete_its_deadline(tmp_path, monkeypatch):
+ seen: list = []
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: seen.append(deadline) or True
+ )
+ result = tmp_path / "collect_result.json"
+ result.write_text(
+ json.dumps(
+ {
+ "kernels": [{"slug": "me/unsloth-t4-ci-nabcdef012345-1111", "verdict": "pass"}],
+ "statuses": [],
+ }
+ ),
+ encoding = "utf-8",
+ )
+ posted = tmp_path / "posted.json"
+ posted.write_text(json.dumps({"ok": [], "failed": []}), encoding = "utf-8")
+ collect.delete_collected(result, posted)
+ assert seen and seen[0] is not None and seen[0] <= time.time() + collect.RELEASE_BUDGET_SEC
+
+
+def test_the_scheduled_collector_uploads_once_and_reports_real_deletions():
+ steps = [(n, s) for _j, n, s in _steps(_wf(COLLECT_WF))]
+ uploads = [
+ s for _n, s in steps if str(s.get("uses", "")).startswith("actions/upload-artifact@")
+ ]
+ assert len(uploads) == 1, "two uploads: the second one fails the job on an artifact outage"
+ assert uploads[0].get("continue-on-error") is True
+ assert "kaggle_collected/**" in uploads[0]["with"]["path"]
+ summary = next(s for n, s in steps if n == "Summarise")
+ assert (
+ "delete_result.json" in summary["run"]
+ ), "the summary reports the collect step's always-false deleted flag"
+
+
+def test_a_kernel_whose_status_record_was_rejected_is_kept(tmp_path, monkeypatch):
+ """The poster refuses a malformed record and posts nothing for it. Keeping
+ only the refused posts released that kernel with no status delivered."""
+ deleted: list[str] = []
+ monkeypatch.setattr(
+ launch, "delete_kernel", lambda slug, deadline = None: deleted.append(slug) or True
+ )
+ slug = "me/unsloth-t4-ci-nabcdef012345-1111"
+ result = tmp_path / "collect_result.json"
+ result.write_text(
+ json.dumps({"kernels": [{"slug": slug, "verdict": "pass"}], "statuses": []}),
+ encoding = "utf-8",
+ )
+ posted = tmp_path / "posted.json"
+ posted.write_text(json.dumps({"ok": [], "failed": [], "invalid": [slug]}), encoding = "utf-8")
+ collect.delete_collected(result, posted)
+ assert deleted == []
+ outcome = json.loads((tmp_path / "delete_result.json").read_text())
+ assert outcome["kept"] == [slug]
+
+
+def test_the_pass_budget_starts_before_the_kernel_listing():
+ """Five slow listing pages under the socket timeout are minutes. A budget
+ started after them is that much later than the job timeout was sized for."""
+ source = (CI_DIR / "collect.py").read_text(encoding = "utf-8")
+ body = source[source.index("def main(") :]
+ assert body.index("deadline = time.time() + BUDGET_SEC") < body.index("ours = find_ours(")
+
+
+def test_two_commits_that_share_eight_characters_resolve_apart_and_post_twice(monkeypatch):
+ """The transition case: an old 8 character slug for commit A and a 12
+ character one for a commit B starting with the same eight. Two commits."""
+ a_full = "abcdef01" + "a" * 32
+ b_full = "abcdef01ffff" + "b" * 28
+ calls: list[list[str]] = []
+
+ def _gh(args):
+ calls.append(list(args))
+ if "/commits/" in args[1]:
+ sha = args[1].rsplit("/", 1)[-1]
+ return (0, a_full if sha == "abcdef01" else b_full, "")
+ return (0, "", "")
+
+ monkeypatch.setattr(post_statuses, "_gh", _gh)
+ statuses = collect.statuses_from(
+ [
+ {
+ "slug": "me/unsloth-t4-ci-nabcdef01-1111",
+ "sha": "abcdef01",
+ "kind": "notebook",
+ "verdict": "pass",
+ "reason": "ok",
+ },
+ {
+ "slug": "me/unsloth-t4-ci-nabcdef01ffff-2222",
+ "sha": "abcdef01ffff",
+ "kind": "notebook",
+ "verdict": "fail",
+ "reason": "red",
+ },
+ ]
+ )
+ outcome = post_statuses.post_all(statuses, "unslothai/unsloth")
+ posts = [c for c in calls if "/statuses/" in c[1]]
+ assert len(posts) == 2
+ assert {c[1].rsplit("/", 1)[-1] for c in posts} == {a_full, b_full}
+ assert len(outcome["ok"]) == 2
+
+
+def test_a_notebook_kernel_without_its_expected_count_is_never_a_pass(tmp_path, monkeypatch):
+ """A notebook kernel predating the expected-count record could have lost
+ four legs and reported one, which against a default of one read as a pass.
+ With the plan unknown a failure still fails, but a clean set is infra."""
+ monkeypatch.setattr(launch, "fetch_evidence", lambda slug, dest, deadline = None: {})
+ monkeypatch.setattr(launch, "delete_kernel", lambda slug, deadline = None: True)
+ api = _StubApi([], {_ENTRY["slug"]: "COMPLETE"})
+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])
+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "infra" and record["expected"] is None, record
+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": False, "payload": "x"}])
+ record = collect.collect_one(api, dict(_ENTRY), tmp_path, expect = 1, max_age_hours = 3.0)
+ assert record["verdict"] == "fail"
+ # Studio kernels carry exactly one report and keep the default.
+ monkeypatch.setattr(launch, "extract_reports", lambda dest: [{"passed": True}])
+ studio = dict(_ENTRY, kind = "studio", slug = "me/unsloth-t4-ci-sabcdef01-1111")
+ api = _StubApi([], {studio["slug"]: "COMPLETE"})
+ assert (
+ collect.collect_one(api, studio, tmp_path, expect = 1, max_age_hours = 3.0)["verdict"] == "pass"
+ )
+
+
+def test_every_gate_budget_covers_the_reaper_window():
+ """Nothing deletes the kernel now, so one that ignores its own timeout bills
+ until the collector reaps it, up to a schedule interval and a job timeout
+ late. The gate admits a run when `remaining >= budget + reserve`, so a
+ budget below that window lets a wedged kernel bill into the reserve."""
+ collect_wf = _wf(COLLECT_WF)
+ job = collect_wf["jobs"]["collect"]
+ on = collect_wf.get("on") or collect_wf.get(True) or {} # PyYAML reads a bare `on:` as True
+ crons = [s["cron"] for s in on.get("schedule") or []]
+ assert crons, "the collector is not scheduled"
+ minutes = sorted(int(m) for m in crons[0].split()[0].split(","))
+ interval_min = max(b - a for a, b in zip(minutes, minutes[1:])) if len(minutes) > 1 else 60
+ source = COLLECT_WF.read_text(encoding = "utf-8")
+ m = re.search(r"--max-age-hours '\$\{\{ inputs\.max_age_hours \|\| '(\d+)' \}\}'", source)
+ assert m, "the collector's default age ceiling is not readable from the workflow"
+ max_age = float(m.group(1))
+ assert max_age == collect.DEFAULT_MAX_AGE_HOURS
+ window = max_age + (interval_min + job["timeout-minutes"]) / 60
+ for path in (NOTEBOOK_WF, STUDIO_WF):
+ budgets = {
+ int(b) for b in re.findall(r"--budget-hours (\d+)", path.read_text(encoding = "utf-8"))
+ }
+ assert len(budgets) == 1, (path.name, budgets)
+ assert (
+ budgets.pop() >= window
+ ), f"{path.name}: a wedged kernel can bill {window:.2f}h before it is reaped"
diff --git a/tests/kaggle/test_launch_cleanup.py b/tests/kaggle/test_launch_cleanup.py
index f05411fb269..81ed9e7ade0 100644
--- a/tests/kaggle/test_launch_cleanup.py
+++ b/tests/kaggle/test_launch_cleanup.py
@@ -227,6 +227,7 @@ def _push_ok(slug: str):
kernel_timeout_sec,
accelerator = "NvidiaTeslaT4",
attempted = None,
+ **kwargs,
):
attempted = [] if attempted is None else attempted
attempted.append(slug)
@@ -481,7 +482,7 @@ def _waiting_launcher(outdir: Path) -> str:
" launch._api = lambda *a, **k: _Api()",
" launch.sweep_orphans = lambda *a, **k: []",
" def _push(notebook, user, kernel_timeout_sec,",
- " accelerator='NvidiaTeslaT4', attempted=None):",
+ " accelerator='NvidiaTeslaT4', attempted=None, **kwargs):",
" attempted = [] if attempted is None else attempted",
" attempted.append('me/k-1')",
" launch._inflight_add('me/k-1')",
@@ -1063,7 +1064,7 @@ def test_a_kernel_pushed_before_the_signal_is_still_deleted(tmp_path):
" launch.sweep_orphans = lambda *a, **k: []",
" calls = []",
" def _push(notebook, user, kernel_timeout_sec,",
- " accelerator='NvidiaTeslaT4', attempted=None):",
+ " accelerator='NvidiaTeslaT4', attempted=None, **kwargs):",
" calls.append(notebook)",
" attempted = [] if attempted is None else attempted",
" slug = 'me/k-%d' % len(calls)",
diff --git a/tests/kaggle/test_studio_gpu_harness.py b/tests/kaggle/test_studio_gpu_harness.py
index f47095a7d15..fc8461a0e83 100644
--- a/tests/kaggle/test_studio_gpu_harness.py
+++ b/tests/kaggle/test_studio_gpu_harness.py
@@ -950,16 +950,20 @@ def test_the_two_legs_together_fit_inside_the_ci_allowance():
def test_the_reserve_leaves_ci_the_fifty_hours_it_is_allowed():
source = WORKFLOW.read_text(encoding = "utf-8")
assert "--reserve-hours 10" in source
- assert "--budget-hours 2" in source
+ assert "--budget-hours 4" in source
+ assert "--budget-hours 2" not in source
-def test_the_budget_hours_flag_covers_the_kernel_ceiling():
- """budget-hours is the worst case one invocation can cost, and the
- kernel ceiling is what enforces that worst case."""
+def test_the_budget_hours_flag_covers_the_reaper_window():
+ """budget-hours is the worst case one invocation can cost. Nothing waits on
+ the kernel now, so a kernel that ignores its own timeout is left to the
+ collector's reaper window rather than to the kernel ceiling."""
source = WORKFLOW.read_text(encoding = "utf-8")
assert "--kernel-timeout-sec 4200" in source
ceiling_hours = 4200 / 3600
- assert ceiling_hours <= 2.0
+ budgets = {int(b) for b in re.findall(r"--budget-hours (\d+)", source)}
+ assert len(budgets) == 1, budgets
+ assert budgets.pop() >= ceiling_hours
def test_the_opt_in_label_the_summary_names_is_the_one_the_gate_reads():
@@ -1671,6 +1675,44 @@ def test_a_nested_executed_notebook_is_read_too(tmp_path):
assert total and len(chunks) == total
+def test_evidence_that_is_json_but_not_a_notebook_does_not_raise(tmp_path):
+ """`[]`, a string cell, a numeric output: all valid JSON, all seen back from
+ a kernel. The reader used to raise on the first `.get`, failing the job
+ after its verdict was already posted."""
+ evidence = tmp_path / "evidence" / "unsloth-t4-ci-1234"
+ evidence.mkdir(parents = True)
+ (evidence / "a_output.ipynb").write_text("[]", encoding = "utf-8")
+ (evidence / "b_output.ipynb").write_text(
+ json.dumps({"cells": ["not a cell", {"outputs": [7, {"text": ["x", 1]}]}]}),
+ encoding = "utf-8",
+ )
+ blob = _bundle({"studio.log": b"a log"})
+ (evidence / "c_output.ipynb").write_text(
+ json.dumps({"cells": [{"outputs": [{"text": "\n".join(_chunk_lines(blob))}]}]}),
+ encoding = "utf-8",
+ )
+ streams = list(collect_evidence.iter_text(tmp_path / "evidence"))
+ assert "x1" in streams
+ chunks, total = collect_evidence.collect_chunks(iter(streams))
+ assert total and len(chunks) == total
+
+
+def test_the_evidence_unpack_is_best_effort_on_the_gpu_job():
+ """The notebook and scheduled unpacks already are; this one decided the
+ colour of a job whose verdict had already been posted."""
+ yaml = pytest.importorskip("yaml")
+ workflow = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))
+ steps = [
+ s
+ for job in workflow["jobs"].values()
+ for s in job.get("steps", [])
+ if "collect_evidence.py" in (s.get("run") or "")
+ ]
+ assert steps, "no unpack step"
+ for step in steps:
+ assert step.get("continue-on-error") is True, step.get("name")
+
+
# ------------------------------------------------------- a diverged training run
diff --git a/tests/kaggle/test_t4_ci_transport.py b/tests/kaggle/test_t4_ci_transport.py
index 896d1895948..3e7cf25a374 100644
--- a/tests/kaggle/test_t4_ci_transport.py
+++ b/tests/kaggle/test_t4_ci_transport.py
@@ -1171,6 +1171,7 @@ def _drive_main(
kernel_timeout_sec,
accelerator = "NvidiaTeslaT4",
attempted = None,
+ **kwargs,
):
clock["t"] += push_seconds
outcome = outcomes.pop(0)
@@ -1634,6 +1635,7 @@ def _accepting_push(slug: str):
kernel_timeout_sec,
accelerator = "NvidiaTeslaT4",
attempted = None,
+ **kwargs,
):
if attempted is not None:
attempted.append(slug)
@@ -2330,7 +2332,7 @@ def test_main_bounds_the_whole_evidence_phase_it_is_budgeted_for(monkeypatch, tm
monkeypatch.setattr(
launch,
"push",
- lambda nb, user, t, accelerator = "NvidiaTeslaT4", attempted = None: (
+ lambda nb, user, t, accelerator = "NvidiaTeslaT4", attempted = None, **kwargs: (
attempted.append(f"{user}/s{len(attempted)}"),
{"ok": True, "slug": attempted[-1], "attempts": list(attempted)},
)[1],
diff --git a/tests/kaggle/test_t4_smoke_harness.py b/tests/kaggle/test_t4_smoke_harness.py
index 91e4a42655f..5811d6bc9ae 100644
--- a/tests/kaggle/test_t4_smoke_harness.py
+++ b/tests/kaggle/test_t4_smoke_harness.py
@@ -591,13 +591,23 @@ def test_the_gate_job_deadline_exceeds_the_gates_own_bound():
"""
import gate
- worst = 3 * gate.SOCKET_TIMEOUT_SEC + gate.SURVEY_BUDGET_SEC
- before_the_gate = 120
- timeout_s = _workflow()["jobs"]["gate"]["timeout-minutes"] * 60
- assert timeout_s >= worst + before_the_gate, (
- f"the gate can take {worst}s, the steps before it up to "
- f"{before_the_gate}s, and the job is killed at {timeout_s}s"
+ # Per account: authentication, username and quota, each at the socket
+ # ceiling. Then ONE survey budget shared across every account (two back to
+ # back would be the bug), and the call still in flight when it expires.
+ accounts = len(gate.DEFAULT_ACCOUNT_ENVS)
+ worst = (
+ accounts * 3 * gate.SOCKET_TIMEOUT_SEC + gate.SURVEY_BUDGET_SEC + gate.SOCKET_TIMEOUT_SEC
)
+ before_the_gate = 120
+ for path in (WORKFLOW, WORKFLOW.parent / "kaggle-t4-studio-gpu-ci.yml"):
+ workflow = pytest.importorskip("yaml").safe_load(path.read_text(encoding = "utf-8"))
+ timeout_s = workflow["jobs"]["gate"]["timeout-minutes"] * 60
+ assert timeout_s >= worst + before_the_gate, (
+ f"{path.name}: the gate can take {worst}s, the steps before it up to "
+ f"{before_the_gate}s, and the job is killed at {timeout_s}s"
+ )
+ source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")
+ assert "survey_deadline - time.monotonic()" in source, "the surveys do not share one budget"
def test_an_unsampled_invocation_is_a_skip_not_a_failure(monkeypatch, tmp_path):
@@ -672,7 +682,7 @@ def _run_gate_against(
monkeypatch.setattr(gate, "client_username", lambda api: "danielhanchen")
monkeypatch.setattr(gate, "remaining_gpu_hours", lambda api: dict(quota))
monkeypatch.setattr(
- gate, "survey_kernels", answer if callable(answer) else (lambda api: answer)
+ gate, "survey_kernels", answer if callable(answer) else (lambda api, **k: answer)
)
code, outputs = _run_gate(monkeypatch, tmp_path, "--force", "true", *extra)
summary_path = tmp_path / "summary.md"
@@ -2732,15 +2742,20 @@ def test_a_dispatched_ref_is_resolved_to_one_commit():
than a commit, so the drift is invisible afterwards too. Same hazard as the
zoo pin below, same answer.
"""
- steps = _workflow()["jobs"]["t4-smoke"]["steps"]
+ workflow = _workflow()
+ steps = workflow["jobs"]["t4-smoke"]["steps"]
names = [s.get("name") for s in steps]
ref = steps[names.index("Resolve the ref under test")]
assert ref["env"]["UNSLOTH_REF"] == "${{ inputs.unsloth_ref }}"
- # Resolved once, here, and retried before it gives up.
- assert "git ls-remote https://github.com/unslothai/unsloth" in ref["run"]
- assert "for attempt in 1 2 3" in ref["run"]
+ # Resolved once, by the GATE, and retried there; this job takes that answer
+ # rather than asking a moving ref again.
+ gate_ref = next(s for s in workflow["jobs"]["gate"]["steps"] if s.get("id") == "ref")
+ assert "git ls-remote https://github.com/unslothai/unsloth" in gate_ref["run"]
+ assert "for attempt in 1 2 3" in gate_ref["run"]
# A full commit needs no resolving and ls-remote would not answer for one.
- assert "^[0-9a-f]{40}$" in ref["run"]
+ assert "^[0-9a-f]{40}$" in gate_ref["run"]
+ assert ref["env"]["GATE_SHA"] == "${{ needs.gate.outputs.head_sha }}"
+ assert "$(git ls-remote" not in ref["run"]
# A ref that cannot be pinned stands the run down rather than installing
# a moving branch, exactly as an unresolvable zoo commit does.
assert "stand_down=true" in ref["run"]
@@ -2762,7 +2777,7 @@ def test_the_resolve_step_pins_every_shape_of_ref_it_can_be_given(tmp_path):
thing, and this one has four branches: no input, a mutable branch or tag, a
commit needing no resolution, and a ref that resolves to nothing.
"""
- steps = _workflow()["jobs"]["t4-smoke"]["steps"]
+ steps = _workflow()["jobs"]["gate"]["steps"]
script = next(s for s in steps if s.get("id") == "ref")["run"]
def drive(
@@ -2794,25 +2809,69 @@ def test_the_resolve_step_pins_every_shape_of_ref_it_can_be_given(tmp_path):
return dict(line.split("=", 1) for line in out.read_text().splitlines() if "=" in line)
# No input: the head SHA under test, which is also the tree checked out.
- assert drive("", "") == {"ref": "headsha"}
+ assert drive("", "") == {"head_sha": "headsha"}
# A branch resolves to the commit it points at, once, for all four legs.
main_sha = "dead" + "0" * 36
- assert drive("main", f"{main_sha}\trefs/heads/main\n") == {"ref": main_sha}
+ assert drive("main", f"{main_sha}\trefs/heads/main\n") == {"head_sha": main_sha}
# An annotated tag lists the tag object first; the COMMIT is what pip can
# check out, and it is on the ^{} line.
tag, commit = "aaaa" + "0" * 36, "bbbb" + "0" * 36
assert drive("v1.2", f"{tag}\trefs/tags/v1.2\n{commit}\trefs/tags/v1.2^{{}}\n") == {
- "ref": commit
+ "head_sha": commit
}
# A full commit is already immutable, and ls-remote answers nothing for
# one, so it must not be sent there at all.
- assert drive("f" * 40, "") == {"ref": "f" * 40}
- # Nothing resolved: stand down rather than install a moving branch.
- assert drive("no-such-branch", "") == {"stand_down": "true"}
+ assert drive("f" * 40, "") == {"head_sha": "f" * 40}
+ # Nothing resolved: an EMPTY key, which the GPU job turns into a stand-down.
+ assert drive("no-such-branch", "") == {"head_sha": ""}
# The value is free text from a dispatch form and reaches no shell.
- assert drive("'; touch pwned; echo '", "") == {"stand_down": "true"}
+ assert drive("'; touch pwned; echo '", "") == {"head_sha": ""}
assert not (tmp_path / "pwned").exists()
+ # The GPU job's step takes the gate's answer and never resolves again.
+ gpu_script = next(s for s in _workflow()["jobs"]["t4-smoke"]["steps"] if s.get("id") == "ref")[
+ "run"
+ ]
+
+ def drive_gpu(
+ unsloth_ref,
+ gate_sha,
+ fetch_exit = 0,
+ ):
+ work = tmp_path / f"gpu{abs(hash((unsloth_ref, gate_sha, fetch_exit)))}"
+ stub = work / "bin"
+ stub.mkdir(parents = True)
+ (stub / "git").write_text(
+ "#!/bin/sh\n"
+ 'case "$1" in\n'
+ ' ls-remote) echo "ls-remote must not be called" >&2; exit 97 ;;\n'
+ ' fetch) exit "$GIT_FETCH_EXIT" ;;\n'
+ " *) exit 0 ;;\n"
+ "esac\n"
+ )
+ (stub / "sleep").write_text("#!/bin/sh\nexit 0\n")
+ for name in ("git", "sleep"):
+ (stub / name).chmod(0o755)
+ out = work / "github_output"
+ out.write_text("", encoding = "utf-8")
+ env = dict(
+ os.environ,
+ PATH = f"{stub}:{os.environ['PATH']}",
+ GITHUB_OUTPUT = str(out),
+ UNSLOTH_REF = unsloth_ref,
+ HEAD_SHA = "headsha",
+ GATE_SHA = gate_sha,
+ GIT_FETCH_EXIT = str(fetch_exit),
+ )
+ done = subprocess.run(["bash", "-c", gpu_script], env = env, capture_output = True, text = True)
+ assert done.returncode == 0, done.stderr
+ return dict(line.split("=", 1) for line in out.read_text().splitlines() if "=" in line)
+
+ assert drive_gpu("", "") == {"ref": "headsha"}
+ assert drive_gpu("main", main_sha) == {"ref": main_sha}
+ assert drive_gpu("main", "") == {"stand_down": "true"}
+ assert drive_gpu("main", main_sha, fetch_exit = 128) == {"stand_down": "true"}
+
def test_the_harness_stays_on_the_checked_out_tree_when_a_ref_is_dispatched():
"""Resolving the INSTALL is not the same as checking that ref out.
@@ -3039,10 +3098,11 @@ def test_the_account_is_rechecked_after_the_concurrency_slot_is_held():
# --force skips the sampling draw only; the dice were rolled by the gate.
assert "--force true" in recheck["run"]
assert "--reserve-hours" in recheck["run"] and "--kernels" in recheck["run"]
- # and it is the last thing before the push.
- assert names[names.index("Recheck the Kaggle account") + 2] == "Launch on Kaggle and collect"
- launch = steps[names.index("Launch on Kaggle and collect")]
- assert launch["if"] == "steps.recheck.outputs.should_run == 'true'"
+ # and nothing that spends a session runs unless it approved. On the
+ # dispatch step's condition, not step order: collection now sits between the
+ # two, so a positional rule would assert the file's shape, not the property.
+ dispatch = steps[names.index("Dispatch to Kaggle")]
+ assert "steps.recheck.outputs.should_run == 'true'" in dispatch["if"]
def test_the_exhausted_quota_failure_reaches_the_pull_request():
@@ -3196,8 +3256,8 @@ def test_an_unresolvable_zoo_commit_stands_the_run_down():
# Nothing that spends a kernel runs after a stand-down.
for name in ("Build the kernel notebooks", "Recheck the Kaggle account"):
assert "steps.pins.outputs.stand_down != 'true'" in steps[names.index(name)]["if"]
- launch_step = steps[names.index("Launch on Kaggle and collect")]
- assert launch_step["if"] == "steps.recheck.outputs.should_run == 'true'"
+ launch_step = steps[names.index("Dispatch to Kaggle")]
+ assert "steps.recheck.outputs.should_run == 'true'" in launch_step["if"]
def test_a_step_count_that_could_only_report_red_stands_the_run_down():
@@ -3405,7 +3465,12 @@ def test_an_evidence_upload_outage_cannot_colour_the_check_red():
assert "::warning" in warn["run"]
# The verdict still runs, and still on its own condition.
report = steps[names.index("Report")]
- assert report["if"] == "always() && steps.recheck.outputs.should_run == 'true'"
+ # always(), so a failed upload still reaches the verdict, and gated on "there
+ # is evidence to read" rather than "the recheck approved": a dispatching run
+ # approves but produces none, and a reporter over an empty directory prints
+ # "0 of 5 payloads", which reads as failure rather than pending.
+ assert report["if"].startswith("always()")
+ assert "hashFiles('kaggle_evidence/**/*_output.ipynb')" in report["if"]
assert "steps.evidence" not in report["if"]
@@ -4602,7 +4667,8 @@ def test_a_dispatched_commit_is_proven_to_exist_before_the_quota_is_spent(tmp_pa
GITHUB_OUTPUT = str(out),
UNSLOTH_REF = unsloth_ref,
HEAD_SHA = "headsha",
- LS_OUT = ls_remote,
+ # What the gate resolved: the GPU step takes it as given.
+ GATE_SHA = ls_remote.split("\t", 1)[0] if ls_remote else unsloth_ref,
GIT_FETCH_EXIT = str(fetch_exit),
FETCH_LOG = str(log),
)
diff --git a/tests/kaggle/test_two_account_selection.py b/tests/kaggle/test_two_account_selection.py
index ce337a9dcda..c52e2a83f97 100644
--- a/tests/kaggle/test_two_account_selection.py
+++ b/tests/kaggle/test_two_account_selection.py
@@ -283,8 +283,30 @@ def test_the_account_draw_is_salted_apart_from_the_sampling_draw():
source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")
picked = source.split("def weighted_pick", 1)[1].split("\ndef ", 1)[0]
assert (
- 'sha256(("account:" + run_id)' in picked
- ), "the account draw hashes the bare run id, which is what sampled_in hashes"
+ 'sha256(("account:" + key)' in picked
+ ), "the account draw hashes the bare key, which is what sampled_in hashes"
+
+
+def test_every_run_of_one_commit_lands_on_the_same_account():
+ """A label or a forced dispatch starts a second run of the SAME commit with
+ a new run id. Keyed on the run id the draw could pick the other account,
+ which cannot see the first's kernel, and dispatch a duplicate."""
+ weights = {"1": 60.0, "2": 30.0}
+ sha = "0123456789abcdef0123456789abcdef01234567"
+ assert len({gate.weighted_pick(sha, weights)[0] for _ in range(5)}) == 1
+ source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")
+ assert (
+ "account_key = (args.head_sha or" in source
+ ), "the gate does not key the draw on the commit"
+ for path in (NOTEBOOK_WF, STUDIO_WF):
+ gate_steps = [
+ s
+ for _j, _n, s in _steps(_wf(path))
+ if "gate.py" in (s.get("run") or "") and "--percent" in s["run"]
+ ]
+ assert gate_steps, f"{path.name} has no gate step"
+ for step in gate_steps:
+ assert "--head-sha" in step["run"], f"{path.name}: the gate is not told the commit"
def test_an_account_with_no_readable_quota_gets_no_weight_but_keeps_its_turn():
@@ -452,3 +474,251 @@ def test_neither_token_name_can_reach_the_kernel():
assert (
f'"{name}"' in source
), f"{name} is not in the credential-leak guard, so a kernel could carry it"
+
+
+def test_a_kernel_already_running_this_commit_on_any_account_stands_the_run_down():
+ """A handover lands a retry on the other account, whose GPU job collects
+ with only its own token and sees nothing in flight. So the gate asks every
+ account it surveys before it picks one."""
+ sha = "abcdef0123456789" + "0" * 24
+ own = [
+ "danielhanchen/unsloth-t4-ci-sabcdef012345-1111 (RUNNING)", # studio, this commit
+ "danielhanchen/unsloth-t4-ci-nffffffffffff-2222 (RUNNING)", # notebook, other commit
+ "danielhanchen/unsloth-t4-ci-nabcdef01-3333 (QUEUED)", # notebook, old slug form
+ ]
+ assert gate.in_flight_for_commit(own, sha, "notebook") == (
+ "danielhanchen/unsloth-t4-ci-nabcdef01-3333"
+ )
+ assert gate.in_flight_for_commit(own, sha, "studio") == (
+ "danielhanchen/unsloth-t4-ci-sabcdef012345-1111"
+ )
+ assert gate.in_flight_for_commit(own, "1234567890ab", "notebook") is None
+ assert gate.in_flight_for_commit(own, sha, "") is None
+ assert gate.in_flight_for_commit(["someone/unsloth-probe-x (RUNNING)"], sha, "notebook") is None
+ source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")
+ main = source[source.index("def main(") :]
+ # Every candidate is asked BEFORE any is chosen: the first loop over `order`
+ # is the in-flight sweep, the selection loop comes after it.
+ sweep, selection = main.split("for account_id in order:", 2)[1:]
+ assert "in_flight_for_commit(survey" in sweep and "concurrency_verdict(" not in sweep
+ assert "concurrency_verdict(" in selection and "in_flight_for_commit(" not in selection
+ assert (
+ "_survey(account_id)" in sweep and "_survey(account_id)" in selection
+ ), "surveys are not shared between the sweep and the selection"
+ for path, kind in ((NOTEBOOK_WF, "notebook"), (STUDIO_WF, "studio")):
+ gate_steps = [
+ s
+ for _j, _n, s in _steps(_wf(path))
+ if "gate.py" in (s.get("run") or "") and "--percent" in s["run"]
+ ]
+ assert gate_steps, path.name
+ for step in gate_steps:
+ assert f"--kind {kind}" in step["run"], f"{path.name}: the gate is not told its kind"
+
+
+def _drive_gate(
+ monkeypatch,
+ tmp_path,
+ *,
+ holder,
+ outcomes = None,
+ extra = (),
+ clock = None,
+ holder_slot = "1",
+):
+ """Run gate.main() with two stub accounts. `holder` is the account whose
+ survey shows a notebook kernel of the commit under test; `outcomes` maps an
+ account to a probe outcome other than ok, the client still handed back."""
+ sha = "abcdef0123456789" + "0" * 24
+ outcomes = outcomes or {}
+ apis = {"1": object(), "2": object()}
+
+ def probe(account_id, env_name, **_kw):
+ quota = {"total_hours": 60.0 if account_id == "1" else 30.0, "remaining_hours": 50.0}
+ record = {
+ "account": account_id,
+ "env": env_name,
+ "outcome": outcomes.get(account_id, "ok"),
+ "user": f"user{account_id}",
+ "quota": quota,
+ "total_hours": quota["total_hours"],
+ "remaining_hours": 50.0,
+ "reserve_hours": 1.0,
+ }
+ return record, apis[account_id]
+
+ surveys_asked: list[tuple[str, float | None]] = []
+
+ def survey(api, *a, **k):
+ account_id = next(i for i, obj in apis.items() if obj is api)
+ surveys_asked.append((account_id, k.get("budget_sec")))
+ if clock is not None:
+ clock[0] += 170.0 # a slow survey
+ mark = "" if holder_slot == "1" else holder_slot
+ own = (
+ [f"user{holder}/unsloth-t4-ci-n{sha[:12]}-{mark}1111 (RUNNING)"]
+ if account_id == holder
+ else []
+ )
+ return {
+ "busy": list(own),
+ "own": own,
+ "foreign": [],
+ "complete": True,
+ "out_of_budget": False,
+ "surveyed": len(own),
+ "unreadable": 0,
+ "gone": 0,
+ "window_hours": 12,
+ }
+
+ monkeypatch.setattr(gate, "probe_account", probe)
+ monkeypatch.setattr(gate, "survey_kernels", survey)
+ if clock is not None:
+ monkeypatch.setattr(gate.time, "monotonic", lambda: clock[0])
+ monkeypatch.setenv("KAGGLE_API_TOKEN", "x")
+ monkeypatch.setenv("KAGGLE_API_TOKEN_2", "y")
+ monkeypatch.setenv("GITHUB_OUTPUT", str(tmp_path / "out.txt"))
+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "summary.md"))
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "gate.py",
+ "--budget-hours",
+ "1",
+ "--reserve-hours",
+ "1",
+ "--force",
+ "true",
+ "--head-sha",
+ sha,
+ "--kind",
+ "notebook",
+ "--run-id",
+ "7",
+ *extra,
+ ],
+ )
+ code = gate.main()
+ outputs = dict(
+ line.split("=", 1)
+ for line in (tmp_path / "out.txt").read_text().splitlines()
+ if "=" in line
+ )
+ return code, outputs, surveys_asked, sha
+
+
+def _other(sha: str) -> tuple[str, str]:
+ sampled = gate.weighted_pick(sha, {"1": 60.0, "2": 30.0})[0]
+ return sampled, ("2" if sampled == "1" else "1")
+
+
+def test_the_gate_stands_down_when_the_other_account_already_runs_this_commit(
+ monkeypatch, tmp_path
+):
+ """The sampled account is FREE and the other holds a kernel for this commit,
+ which is what a retry after a handover looks like. A gate that only asks the
+ account it is about to pick finds it clear and dispatches a duplicate."""
+ _sampled, other = _other("abcdef0123456789" + "0" * 24)
+ code, outputs, asked, _sha = _drive_gate(monkeypatch, tmp_path, holder = other)
+ assert code == 0
+ assert outputs["should_run"] == "false", outputs["reason"]
+ assert f"account {other}" in outputs["reason"] and "already running" in outputs["reason"]
+ assert {a for a, _b in asked} == {"1", "2"}, "the account holding the kernel was never asked"
+ # One survey per account, reused by the selection loop.
+ assert len(asked) == 2, asked
+
+
+def test_an_account_that_cannot_launch_is_still_asked_whether_it_runs_this_commit(
+ monkeypatch, tmp_path
+):
+ """The account that dispatched this commit is the one likely to be short of
+ quota now. Skipping it in the sweep hands the retry to the other account,
+ which dispatches the same commit again."""
+ _sampled, other = _other("abcdef0123456789" + "0" * 24)
+ for outcome in ("insufficient_quota", "quota_unreadable"):
+ code, outputs, asked, _sha = _drive_gate(
+ monkeypatch, tmp_path, holder = other, outcomes = {other: outcome}
+ )
+ assert outputs["should_run"] == "false", (outcome, outputs["reason"])
+ assert "already running" in outputs["reason"], (outcome, outputs["reason"])
+ assert other in {a for a, _b in asked}, (outcome, asked)
+
+
+def test_the_second_slot_is_not_a_duplicate_but_its_own_retry_is(monkeypatch, tmp_path):
+ """`slot: 2` is a second session on the same ref beside slot 1, so a slot-1
+ kernel must not stand it down, while a retry of the slot-2 run itself must.
+ The slot in the slug tells the two apart."""
+ sampled, _unused = _other("abcdef0123456789" + "0" * 24)
+ # Slot 1 kernel up, slot 2 asked for: runs.
+ _c, outputs, _a, _s = _drive_gate(
+ monkeypatch, tmp_path, holder = sampled, extra = ("--slot", "2"), holder_slot = "1"
+ )
+ assert outputs["should_run"] == "true", outputs["reason"]
+ # Slot 2 kernel up, slot 2 asked for again: stands down.
+ _c, outputs, _a, _s = _drive_gate(
+ monkeypatch, tmp_path, holder = sampled, extra = ("--slot", "2"), holder_slot = "2"
+ )
+ assert outputs["should_run"] == "false", outputs["reason"]
+ assert "slot 2" in outputs["reason"]
+ # Slot 2 kernel up, slot 1 asked for: runs, it is the other seat.
+ _c, outputs, _a, _s = _drive_gate(
+ monkeypatch, tmp_path, holder = sampled, extra = ("--slot", "1"), holder_slot = "2"
+ )
+ assert outputs["should_run"] == "true", outputs["reason"]
+ # And slot 1 against slot 1 still stands down.
+ _c, outputs, _a, _s = _drive_gate(monkeypatch, tmp_path, holder = sampled, holder_slot = "1")
+ assert outputs["should_run"] == "false", outputs["reason"]
+ # The workflow threads its slot input through the gate, the collector's
+ # in-flight check and the launcher (which writes it into the slug).
+ steps = {s.get("id"): s for _j, _n, s in _steps(_wf(NOTEBOOK_WF)) if s.get("id")}
+ for step_id, script in (
+ ("decide", "gate.py"),
+ ("collect", "collect.py"),
+ ("launch", "launch.py"),
+ ):
+ run = steps[step_id]["run"]
+ assert script in run
+ assert "--slot '${{ inputs.slot || '1' }}'" in run, f"{step_id} is not told the slot"
+ assert "--sha '${{ steps.ref.outputs.ref }}'" in steps["collect"]["run"]
+
+
+def test_the_surveys_share_one_budget(monkeypatch, tmp_path):
+ """Two surveys each bounded by SURVEY_BUDGET_SEC outlive the gate job back
+ to back. The second gets whatever the first left of ONE budget."""
+ clock = [1000.0]
+ _sampled, other = _other("abcdef0123456789" + "0" * 24)
+ _code, _outputs, asked, _sha = _drive_gate(monkeypatch, tmp_path, holder = other, clock = clock)
+ assert len(asked) == 2, asked
+ first, second = asked[0][1], asked[1][1]
+ assert first == pytest.approx(gate.SURVEY_BUDGET_SEC)
+ assert second == pytest.approx(gate.SURVEY_BUDGET_SEC - 170.0), (first, second)
+
+
+def test_the_gate_is_keyed_on_the_commit_the_gpu_job_will_test():
+ """A dispatch naming `unsloth_ref` tests that ref, not github.sha. Keyed on
+ github.sha the gate looks for the wrong kernel, and once the default branch
+ moves two runs of one ref can land on different accounts."""
+ for path in (NOTEBOOK_WF, STUDIO_WF):
+ gate_job = _wf(path)["jobs"]["gate"]
+ steps = {s.get("id"): s for s in gate_job["steps"] if s.get("id")}
+ ref = steps.get("ref")
+ assert ref is not None, f"{path.name}: the gate job does not name the commit under test"
+ assert ref["env"]["UNSLOTH_REF"] == "${{ inputs.unsloth_ref }}"
+ assert "head_sha=" in ref["run"] and "git ls-remote" in ref["run"]
+ decide = steps["decide"]
+ assert (
+ "--head-sha '${{ steps.ref.outputs.head_sha }}'" in decide["run"]
+ ), f"{path.name}: the gate is keyed on a commit the GPU job may not test"
+ assert "github.sha" not in decide["run"]
+ # And the GPU job takes the gate's answer rather than resolving a
+ # moving ref a second time.
+ assert gate_job["outputs"]["head_sha"] == "${{ steps.ref.outputs.head_sha }}"
+ gpu = next(
+ s
+ for _j, _n, s in _steps(_wf(path))
+ if s.get("id") == "ref" and "GATE_SHA" in (s.get("env") or {})
+ )
+ assert gpu["env"]["GATE_SHA"] == "${{ needs.gate.outputs.head_sha }}"
+ assert "$(git ls-remote" not in gpu["run"]