Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kaggle CI: use both accounts, weighted by their own weekly quota (#10121)

Daniel Handddf214b105d
diff --git a/.github/scripts/kaggle_t4_ci/gate.py b/.github/scripts/kaggle_t4_ci/gate.py
index 3254f5898fc..efe1033dedb 100644
--- a/.github/scripts/kaggle_t4_ci/gate.py
+++ b/.github/scripts/kaggle_t4_ci/gate.py
@@ -3,10 +3,10 @@
"""Decide whether this invocation is allowed to spend Kaggle GPU quota.
-The account behind KAGGLE_ACCESS_TOKEN_GH has a WEEKLY accelerator budget shared
-with every other use of that account, and a kernel on every push would drain a
-week of it in a day. So the default answer is "no" and a job earns "yes" through
-four checks, in this order:
+Each Kaggle account CI may spend has a WEEKLY accelerator budget shared with
+every other use of that account, and a kernel on every push would drain a week
+of it in a day. So the default answer is "no" and a job earns "yes" through five
+checks, in this order:
0. **Was this event a request at all?** A run started by APPLYING a label is a
request only when that label is the opt-in one. The trigger fires on all
@@ -17,10 +17,23 @@ four checks, in this order:
2. **Sampling.** Roughly one invocation in ten, derived from the run id so a
re-run of the same run gives the same answer (otherwise anyone could reroll
until it fires) while different runs stay independent.
-3. **Remaining quota.** Refuses when what is left would not cover this
+3. **Which account.** There is more than one, with different weekly
+ allowances, and traffic is split in proportion to them: an account with
+ twice the hours takes twice the runs. The weights are READ from each
+ account's own ``quota_view()`` rather than written down here, so a plan
+ changed on Kaggle's side changes the split without anyone editing this file.
+ The draw is keyed on the run id ALONE -- not the attempt -- so a re-run
+ picks the SAME account as the attempt whose kernels may still be in flight.
+ An account that cannot take the run (no quota, no free session, no readable
+ answer) hands over to the other rather than standing the run down; see
+ FALLBACK_ELIGIBLE.
+4. **Remaining quota.** Refuses when what is left would not cover this
invocation's worst case plus a reserve, so CI never drains the account. This
- is the ONE stand-down that is a failure rather than a skip; see below.
-4. **Concurrency.** Kaggle caps concurrent batch GPU kernels at 2, per ACCOUNT
+ is the ONE stand-down that is a failure rather than a skip; see below. The
+ reserve SCALES with the account: a flat 20h held back from a 30h account is
+ two thirds of it and one third of a 60h one, which would quietly make the
+ smaller account the stricter one. See ``scaled_reserve``.
+5. **Concurrency.** Kaggle caps concurrent batch GPU kernels at 2, per ACCOUNT
rather than per workflow. The survey splits kernels this workflow pushed
from the rest because they call for opposite answers: a FOREIGN kernel means
a human is on the shared account and this invocation stands down entirely,
@@ -183,6 +196,41 @@ QUOTA_EXHAUSTED_MESSAGE = (
"GPU capacity exhausted - please wait until next week - you can ignore this CI failure"
)
+# The accounts CI may spend, in order. The env var names are the SECRET names
+# the workflows pass through; the account id is the position, which is what the
+# concurrency group and the job summary use.
+#
+# Deliberately not carrying each account's weekly hours: they are read from
+# `quota_view()` per run, so a plan changed on Kaggle's side changes the traffic
+# split with nothing here to update and nothing here to go stale.
+DEFAULT_ACCOUNT_ENVS = ("KAGGLE_API_TOKEN", "KAGGLE_API_TOKEN_2")
+
+# `--reserve-hours` was calibrated against an account of this size. See
+# `scaled_reserve`: the reserve is a FRACTION of the plan, not a fixed number of
+# hours, or the smaller account keeps back proportionally more.
+DEFAULT_RESERVE_BASIS_HOURS = 60.0
+
+# An account answering with one of these hands over to the next one instead of
+# standing the whole run down: none of them says anything about the code under
+# test, and the other account may be perfectly able to run it.
+#
+# `credential_absent` is here so a repo holding one secret still works, which is
+# also what a fork sees. What is NOT here is every decision made ABOVE the
+# account -- not sampled, wrong label, bad input -- because those are answers
+# about the run, and trying a second account would be answering a question
+# nobody asked.
+FALLBACK_ELIGIBLE = frozenset(
+ {
+ "credential_absent",
+ "auth_failed",
+ "username_unreadable",
+ "quota_unreadable",
+ "insufficient_quota",
+ "capacity_occupied",
+ "capacity_unreadable",
+ }
+)
+
def _looks_gone(exc: BaseException) -> bool:
"""Did this status lookup fail because the kernel no longer exists?"""
@@ -232,14 +280,51 @@ def sampled_in(run_id: str, percent: int) -> tuple[bool, int]:
return draw < percent, draw
-def kaggle_client():
+def kaggle_client(token: str | None = None):
+ """An authenticated client, optionally for a NAMED account's token.
+
+ `authenticate()` reads the credential from the environment, so selecting an
+ account means putting that account's token there for the length of the call
+ and putting back whatever was there before. The restore is not tidiness: the
+ launcher further down this job reads `KAGGLE_API_TOKEN` for itself, and a
+ probe that left the last-tried account's token behind would hand the pushing
+ step a different account from the one this gate cleared.
+ """
from kaggle.api.kaggle_api_extended import KaggleApi
- api = KaggleApi()
- api.authenticate()
+ if token is None:
+ api = KaggleApi()
+ api.authenticate()
+ return api
+
+ previous = os.environ.get("KAGGLE_API_TOKEN")
+ os.environ["KAGGLE_API_TOKEN"] = token
+ try:
+ api = KaggleApi()
+ api.authenticate()
+ finally:
+ if previous is None:
+ os.environ.pop("KAGGLE_API_TOKEN", None)
+ else:
+ os.environ["KAGGLE_API_TOKEN"] = previous
return api
+def client_username(api) -> str | None:
+ """The account this client authenticated AS, from the client's own record.
+
+ `_authenticate_with_access_token` introspects the token and stores the name
+ it comes back with, so this is Kaggle's answer rather than ours. It is not a
+ nicety: a kernel id carries its owner, so the pushing step needs the real
+ username, and a hardcoded one silently belongs to whichever account happened
+ to be first when it was written.
+ """
+ try:
+ return api.config_values.get(api.CONFIG_NAME_USER) or None
+ except Exception: # noqa: BLE001
+ return None
+
+
def remaining_gpu_hours(api) -> dict:
"""Exact remaining accelerator hours from Kaggle's own quota API."""
resp = api.quota_view()
@@ -258,6 +343,122 @@ def remaining_gpu_hours(api) -> dict:
}
+def scaled_reserve(reserve_hours: float, total_hours: float, basis_hours: float) -> float:
+ """The reserve this account keeps for humans, in proportion to its size.
+
+ `--reserve-hours` was calibrated against one account. Applied flat to a
+ smaller one it is a much bigger bite: 20h held out of a 30h plan is two
+ thirds of it against one third of a 60h plan, so the SMALLER account would
+ silently become the stricter one and take even less traffic than its weight
+ already gives it. Scaling keeps "what fraction is kept back" the constant,
+ which is what the number was chosen to express.
+ """
+ if basis_hours <= 0 or total_hours <= 0:
+ return reserve_hours
+ return round(reserve_hours * (total_hours / basis_hours), 3)
+
+
+def weighted_pick(run_id: str, weights: dict[str, float]) -> tuple[str, float]:
+ """Deterministic weighted choice of account, keyed on the run id ALONE.
+
+ Not the attempt, for the same reason `sampled_in` excludes it and a sharper
+ one: a re-run of a run whose kernels are still in flight must return to the
+ SAME account, or the second attempt pushes to an account the first one is
+ not watching and the first one's kernels are reaped by nobody.
+
+ Salted differently from `sampled_in` so the two draws off one run id are
+ independent -- without the salt, whether a run is sampled in would correlate
+ with which account it lands on, and one account would quietly get a
+ different SHARE of the forced runs than of the sampled ones.
+ """
+ ids = sorted(weights)
+ total = sum(max(0.0, weights[i]) for i in ids)
+ digest = hashlib.sha256(("account:" + run_id).encode("utf-8")).hexdigest()
+ draw = (int(digest[:8], 16) % 1_000_000) / 1_000_000.0
+ if not ids or total <= 0:
+ return (ids[0] if ids else ""), draw
+ cumulative = 0.0
+ for account_id in ids:
+ cumulative += max(0.0, weights[account_id]) / total
+ if draw < cumulative:
+ return account_id, draw
+ return ids[-1], draw
+
+
+def probe_account(
+ account_id: str,
+ env_name: str,
+ *,
+ budget_hours: float,
+ reserve_hours: float,
+ reserve_basis_hours: float,
+) -> tuple[dict, object]:
+ """What this account can tell us before a single kernel is pushed.
+
+ Returns (record, client). The record is JSON-safe and goes in the log and
+ the job summary; the client is kept only in memory, for the survey that runs
+ on whichever account is actually chosen.
+
+ Every unusable answer gets a CODE rather than a sentence, because the
+ fallback branches on it. Parsing the human-readable reason would make the
+ wording load bearing, and the wording is written for a person who did not
+ cause the problem and cannot fix it.
+ """
+ record: dict = {"account": account_id, "env": env_name, "outcome": "ok"}
+ token = os.environ.get(env_name)
+ if not token:
+ record["outcome"] = "credential_absent"
+ record["detail"] = (
+ f"{env_name} is not available to this context (expected on a fork "
+ "pull request, where secrets are withheld)"
+ )
+ return record, None
+
+ try:
+ api = kaggle_client(token)
+ except BaseException as exc: # noqa: BLE001
+ if isinstance(exc, KeyboardInterrupt):
+ raise
+ record["outcome"] = "auth_failed"
+ record["error"] = type(exc).__name__
+ record["detail"] = f"could not authenticate to Kaggle: {type(exc).__name__}"
+ return record, None
+
+ # Refused rather than defaulted. The launcher names the kernel's owner, and
+ # a run that cannot say who it authenticated as cannot name it correctly --
+ # which is exactly the state a hardcoded username hides.
+ username = client_username(api)
+ if not username:
+ record["outcome"] = "username_unreadable"
+ record["detail"] = (
+ "could not determine which Kaggle account this token belongs to, so "
+ "the kernel could not be pushed under an owner that can delete it"
+ )
+ return record, None
+ record["user"] = username
+
+ try:
+ quota = remaining_gpu_hours(api)
+ except Exception as exc: # noqa: BLE001
+ quota = {"ok": False, "error": type(exc).__name__}
+ if not quota.get("ok"):
+ record["outcome"] = "quota_unreadable"
+ record["error"] = quota.get("error")
+ record["detail"] = (
+ "could not read the Kaggle accelerator quota, so the remaining budget is unknown"
+ )
+ return record, api
+
+ record["quota"] = quota
+ record["total_hours"] = quota["total_hours"]
+ record["remaining_hours"] = quota["remaining_hours"]
+ reserve = scaled_reserve(reserve_hours, quota["total_hours"], reserve_basis_hours)
+ record["reserve_hours"] = reserve
+ if quota["remaining_hours"] < budget_hours + reserve:
+ record["outcome"] = "insufficient_quota"
+ return record, api
+
+
def _as_naive_utc(value):
"""Kaggle's timestamps, normalised so they can be compared at all."""
if not isinstance(value, datetime):
@@ -486,7 +687,22 @@ def main() -> int:
"--reserve-hours",
type = float,
default = 6.0,
- help = "quota CI refuses to dip into, left for humans",
+ help = "quota CI refuses to dip into, left for humans. Scaled to each "
+ "account's own weekly total against --reserve-basis-hours",
+ )
+ ap.add_argument(
+ "--reserve-basis-hours",
+ type = float,
+ default = DEFAULT_RESERVE_BASIS_HOURS,
+ help = "the account size --reserve-hours was calibrated against, so the "
+ "reserve stays the same FRACTION of a smaller plan",
+ )
+ ap.add_argument(
+ "--account-env",
+ action = "append",
+ default = None,
+ help = "env var holding an account's token; repeat for each account, in "
+ f"order. Default: {', '.join(DEFAULT_ACCOUNT_ENVS)}",
)
ap.add_argument(
"--kernels",
@@ -593,12 +809,13 @@ def main() -> int:
f"invocations",
)
- if not os.environ.get("KAGGLE_API_TOKEN"):
+ account_envs = [e.strip() for e in (args.account_env or DEFAULT_ACCOUNT_ENVS) if e.strip()]
+ if not any(os.environ.get(e) for e in account_envs):
return _decide(
False,
- "KAGGLE_API_TOKEN is not available to this "
- "context (expected on a fork pull request, "
- "where secrets are withheld)",
+ "no Kaggle credential is available to this context (expected on a "
+ "fork pull request, where secrets are withheld). Looked for: "
+ + ", ".join(account_envs),
)
# Before the first network call, and globally: authenticate(), quota_view()
@@ -608,86 +825,190 @@ def main() -> int:
# SOCKET_TIMEOUT_SEC.
socket.setdefaulttimeout(SOCKET_TIMEOUT_SEC)
- try:
- api = kaggle_client()
- except BaseException as exc: # noqa: BLE001
- if isinstance(exc, KeyboardInterrupt):
- raise
- msg = f"could not authenticate to Kaggle: {type(exc).__name__}"
- if not errors_are_skips:
- print(f"[gate] {msg}", flush = True)
- return 1
- return _decide(False, msg)
-
- try:
- quota = remaining_gpu_hours(api)
- except Exception as exc: # noqa: BLE001
- quota = {"ok": False, "error": type(exc).__name__}
- print("[gate] quota " + json.dumps(quota), flush = True)
- _out("quota", json.dumps(quota))
-
- if quota.get("ok"):
- need = args.budget_hours + args.reserve_hours
- if quota["remaining_hours"] < need:
- # THE ONE RED STAND-DOWN, and it is answered here rather than after
- # the survey on purpose: one quota call, no kernel pushed, no
- # session spent to report that there are no sessions left. The
- # required sentence comes first and whole; the numbers behind it
- # follow so the reader can see when the hours come back.
- return _decide(
- False,
- f"{QUOTA_EXHAUSTED_MESSAGE}. "
- f"{quota['remaining_hours']}h of the weekly {quota['total_hours']}h "
- f"accelerator quota is left, and this run needs up to "
- f"{args.budget_hours}h on top of a {args.reserve_hours}h reserve. "
- f"Quota refreshes at {quota.get('refresh_at')}",
- exit_code = 0 if exhaustion_is_soft else 1,
+ # Every account's quota is read BEFORE the draw, because the draw is
+ # weighted by what those calls report. Two quota calls, no session, and the
+ # weights are then a measurement rather than a number somebody typed.
+ probes: dict[str, dict] = {}
+ clients: dict[str, object] = {}
+ for index, env_name in enumerate(account_envs, start = 1):
+ account_id = str(index)
+ try:
+ record, api = probe_account(
+ account_id,
+ env_name,
+ budget_hours = args.budget_hours,
+ reserve_hours = args.reserve_hours,
+ reserve_basis_hours = args.reserve_basis_hours,
)
+ except BaseException as exc: # noqa: BLE001
+ if isinstance(exc, KeyboardInterrupt):
+ raise
+ record, api = (
+ {
+ "account": account_id,
+ "env": env_name,
+ "outcome": "auth_failed",
+ "error": type(exc).__name__,
+ },
+ None,
+ )
+ probes[account_id] = record
+ clients[account_id] = api
+ print("[gate] accounts " + json.dumps(list(probes.values())), flush = True)
+ _out("accounts", json.dumps(list(probes.values())))
+
+ # An account whose quota could not be read has no weight, because a weight
+ # is what its plan says and we did not get to hear it. It stays a CANDIDATE
+ # -- the order below still reaches it -- so an unreadable answer costs the
+ # account its share of the traffic and not its place in the queue.
+ weights = {i: p["total_hours"] for i, p in probes.items() if p.get("total_hours")}
+ sampled_account, account_draw = weighted_pick(args.run_id, weights)
+ if weights:
+ share = max(0.0, weights.get(sampled_account, 0.0)) / sum(weights.values())
+ print(
+ f"[gate] account draw={account_draw:.6f} sampled={sampled_account} "
+ f"p={share:.3f} weights=" + json.dumps({i: weights[i] for i in sorted(weights)}),
+ flush = True,
+ )
else:
- # An unreadable quota is not permission to spend it -- and it is not
- # evidence of exhaustion either, so it stays a green skip.
+ print(
+ f"[gate] account draw={account_draw:.6f} sampled={sampled_account or '(none)'} "
+ "with NO readable weights, so this is the declaration order rather "
+ "than a weighted choice",
+ flush = True,
+ )
+
+ # The sampled account first, then the rest in declaration order. Only the
+ # account actually being considered pays for a survey, which is the
+ # expensive call here.
+ order = [sampled_account] + [i for i in probes if i != sampled_account]
+ handovers: list[str] = []
+
+ for account_id in order:
+ if not account_id:
+ continue
+ record = probes[account_id]
+ if record["outcome"] != "ok":
+ handovers.append(f"account {account_id} {record['outcome']}")
+ continue
+
+ api = clients[account_id]
+ try:
+ survey = survey_kernels(api)
+ except Exception as exc: # noqa: BLE001
+ record["outcome"] = "capacity_unreadable"
+ record["error"] = type(exc).__name__
+ handovers.append(
+ f"account {account_id} kernels could not be listed ({type(exc).__name__})"
+ )
+ continue
+ print(
+ f"[gate] concurrency account={account_id} "
+ + json.dumps(
+ {k: v for k, v in survey.items() if k not in ("busy", "own", "foreign")}
+ | {
+ "busy": len(survey["busy"]),
+ "own": len(survey["own"]),
+ "foreign": len(survey["foreign"]),
+ }
+ ),
+ flush = True,
+ )
+
+ clear, why_not = concurrency_verdict(survey, args.kernels, args.allow_foreign_in_flight)
+ if not clear:
+ record["outcome"] = "capacity_occupied"
+ record["detail"] = why_not
+ handovers.append(f"account {account_id} {why_not}")
+ continue
+
+ quota = record["quota"]
+ _out("quota", json.dumps(quota))
+ _out("account", account_id)
+ _out("account_env", record["env"])
+ _out("account_user", record["user"])
+ # A ONE-ELEMENT matrix, and the token is not in it. The GPU job indexes
+ # the secrets context with `secret_name`, which is the only shape that
+ # cannot end up holding a different account's token than the metadata
+ # beside it claims. See the workflow.
+ _out(
+ "matrix",
+ json.dumps(
+ {
+ "include": [
+ {
+ "account_id": account_id,
+ "secret_name": record["env"],
+ "kaggle_user": record["user"],
+ "weekly_hours": quota["total_hours"],
+ "reserve_hours": record["reserve_hours"],
+ }
+ ]
+ }
+ ),
+ )
+
+ why = (
+ "forced by override"
+ if override
+ else f"sampled in (draw {draw} of 100, threshold {args.percent})"
+ )
+ moved = ""
+ if account_id != sampled_account and handovers:
+ moved = f"; sampled account {sampled_account} handed over ({'; '.join(handovers)})"
return _decide(
- False,
- "could not read the Kaggle accelerator quota, so the remaining budget is unknown",
+ True,
+ f"{why}; running on account {account_id} ({record['user']}) with "
+ f"{quota['remaining_hours']}h of its weekly {quota['total_hours']}h "
+ f"left and {args.kernels} of that account's "
+ f"{MAX_CONCURRENT_GPU_KERNELS} kernel slots free{moved}",
)
- try:
- survey = survey_kernels(api)
- except Exception as exc: # noqa: BLE001
+ # Nobody could run. WHICH stand-down this is depends on what every account
+ # said, and only one of the answers is a failure: an account out of hours is
+ # a fact about the week, while an unreadable one is a fact about the minute.
+ # "Unknown" is not "exhausted", so a single unreadable account keeps the
+ # whole verdict green -- the same rule the one-account gate applied, now over
+ # a set.
+ #
+ # An account with no credential is not a candidate at all. It is what a fork
+ # sees, and what a repo holding one of the two secrets sees, so counting it
+ # as "not exhausted" would turn the exhausted RED into a green skip for
+ # everyone with a single account configured.
+ candidates = {i: p for i, p in probes.items() if p["outcome"] != "credential_absent"}
+ outcomes = {i: p["outcome"] for i, p in candidates.items()}
+
+ if outcomes and all(o == "insufficient_quota" for o in outcomes.values()):
+ detail = "; ".join(
+ f"account {i} has {candidates[i]['remaining_hours']}h of its weekly "
+ f"{candidates[i]['total_hours']}h left against a "
+ f"{candidates[i]['reserve_hours']}h reserve"
+ for i in sorted(candidates)
+ )
+ first = candidates[sorted(candidates)[0]]
return _decide(
False,
- "could not list this account's kernels "
- f"({type(exc).__name__}), so concurrency cannot "
- "be established",
+ f"{QUOTA_EXHAUSTED_MESSAGE}. "
+ f"{detail}, and this run needs up to {args.budget_hours}h on top of "
+ f"that reserve. Quota refreshes at {first['quota'].get('refresh_at')}",
+ exit_code = 0 if exhaustion_is_soft else 1,
)
- print(
- "[gate] concurrency "
- + json.dumps(
- {k: v for k, v in survey.items() if k not in ("busy", "own", "foreign")}
- | {
- "busy": len(survey["busy"]),
- "own": len(survey["own"]),
- "foreign": len(survey["foreign"]),
- }
- ),
- flush = True,
- )
-
- clear, why_not = concurrency_verdict(survey, args.kernels, args.allow_foreign_in_flight)
- if not clear:
- return _decide(False, why_not)
- why = (
- "forced by override"
- if override
- else f"sampled in (draw {draw} of 100, threshold {args.percent})"
- )
- return _decide(
- True,
- f"{why}; {quota['remaining_hours']}h of GPU quota "
- f"remaining and {args.kernels} of the account's "
- f"{MAX_CONCURRENT_GPU_KERNELS} kernel slots are free",
- )
+ if not errors_are_skips and any(
+ o in ("auth_failed", "username_unreadable") for o in outcomes.values()
+ ):
+ print("[gate] no account could be authenticated: " + json.dumps(outcomes), flush = True)
+ return 1
+
+ # The per-account sentences, not the codes: these are read by whoever opened
+ # the pull request, who did not cause any of this and cannot fix it. With one
+ # account configured this reads exactly as the single-account gate did.
+ details = [
+ probes[i].get("detail") or probes[i]["outcome"]
+ for i in sorted(probes)
+ if probes[i]["outcome"] != "ok"
+ ]
+ return _decide(False, "; ".join(details) or "; ".join(handovers))
if __name__ == "__main__":
diff --git a/.github/scripts/kaggle_t4_ci/launch.py b/.github/scripts/kaggle_t4_ci/launch.py
index 0e0dea23177..9ce3eac7e6e 100644
--- a/.github/scripts/kaggle_t4_ci/launch.py
+++ b/.github/scripts/kaggle_t4_ci/launch.py
@@ -320,6 +320,27 @@ def _api():
return api
+def username_of(api) -> str | None:
+ """The account this client authenticated as, as Kaggle itself reports it.
+
+ `_authenticate_with_access_token` introspects the token and records the name
+ that comes back, so this is not a guess. It matters because a kernel id is
+ `<owner>/<slug>`: pushing under a name the token does not own fails, and
+ DELETING under the wrong name silently reaps nothing while the real kernel
+ keeps billing. With more than one account in play the owner cannot be a
+ constant, so it is read from whichever token this process was handed.
+
+ Takes the client the caller already authenticated rather than building one:
+ a second `authenticate()` is a second network round trip on a path that is
+ already bounded, and it could answer for a different token than the one the
+ rest of the run uses.
+ """
+ try:
+ return api.config_values.get(api.CONFIG_NAME_USER) or None
+ except Exception: # noqa: BLE001
+ return None
+
+
# A kernel this process pushed but has not yet deleted is recorded here, so a
# LATER launch can reclaim it. This is the only cover for `kill -9`, where no
# handler of ours ever runs. Deliberately keyed on pid: an entry whose owner
@@ -349,9 +370,22 @@ def _inflight_write(entries: list[dict]) -> None:
pass # bookkeeping only; never fail a run over it
-def _inflight_add(slug: str) -> None:
+def _inflight_add(slug: str, owner: str | None = None) -> None:
+ """File a slug, WITH the account that owns it.
+
+ The owner is recorded because the sweep below runs with whatever token the
+ NEXT job was handed, and CI now hands out more than one. A slug is
+ `<owner>/<name>`, so an entry filed by one account cannot be deleted by the
+ other; without the owner on the record the sweep cannot tell that it is
+ about to try, and a failed delete reads exactly like a kernel that was
+ already gone.
+ """
entries = [e for e in _inflight_read() if e.get("slug") != slug]
- entries.append({"slug": slug, "pid": os.getpid(), "at": time.time()})
+ entry = {"slug": slug, "pid": os.getpid(), "at": time.time()}
+ owner = owner or (slug.split("/", 1)[0] if "/" in slug else None)
+ if owner:
+ entry["owner"] = owner
+ entries.append(entry)
_inflight_write(entries)
@@ -386,11 +420,18 @@ def _pid_alive(pid: int) -> bool:
return True
-def sweep_orphans() -> list[str]:
+def sweep_orphans(owner: str | None = None) -> list[str]:
"""Delete kernels left behind by a launcher that was killed outright.
Only entries whose owning process is gone are eligible, so a concurrent
run is never disturbed. Returns the slugs reclaimed.
+
+ ``owner``, when given, is the account this process authenticated as, and an
+ entry belonging to a DIFFERENT account is left alone rather than attempted.
+ This token cannot delete that kernel, so trying would turn a real leak into
+ a log line indistinguishable from a kernel that was already gone -- and
+ would drop the record, which is the only thing that still knows the kernel
+ exists. Left filed, it is reclaimed by the next run that holds its account.
"""
entries = _inflight_read()
if not entries:
@@ -405,6 +446,14 @@ def sweep_orphans() -> list[str]:
# --keep-kernel asked for this one to stay up.
keep.append(entry)
continue
+ entry_owner = entry.get("owner") or (slug.split("/", 1)[0] if "/" in slug else None)
+ if owner and entry_owner and entry_owner != owner:
+ keep.append(entry)
+ _log(
+ f"leaving orphan {slug} filed: it belongs to {entry_owner} and this "
+ f"job holds {owner}, so this token cannot delete it"
+ )
+ continue
if isinstance(pid, int) and _pid_alive(pid) and pid != os.getpid():
keep.append(entry)
continue
@@ -1070,7 +1119,19 @@ def main() -> int:
help = "kernel notebook to push. Repeatable: all of them "
"are pushed before any of them is waited on",
)
- ap.add_argument("--user", required = True)
+ # NOT required, and not a constant. The owner is a property of the TOKEN
+ # this process was handed, and CI now hands it one of several. Passing the
+ # name is still allowed -- the caller that selected the account knows it --
+ # but it is CHECKED against the token rather than trusted, because the one
+ # failure worth catching here is a name and a credential that disagree: the
+ # push fails, or worse, the cleanup deletes under a name that owns nothing
+ # and the real kernel bills on unwatched.
+ ap.add_argument(
+ "--user",
+ default = "",
+ help = "Kaggle account to push under. Defaults to the account the token "
+ "authenticates as; when given, it must MATCH that account",
+ )
ap.add_argument("--outdir", required = True)
ap.add_argument(
"--expect", type = int, default = 2, help = "payload reports this kernel should produce"
@@ -1239,11 +1300,36 @@ def main() -> int:
result["reason"] = f"kaggle auth failed: {type(exc).__name__}"
return finish()
+ # WHO this token is, settled before anything is pushed and read off the
+ # client just authenticated. CI holds more than one account now, and both
+ # ways to get this wrong are silent: a name the token does not own makes
+ # every push fail for a reason that reads like a bad notebook, and a name
+ # belonging to the OTHER account makes the cleanup delete nothing while the
+ # kernel it filed bills on unwatched.
+ #
+ # Answered through finish() rather than a bare return, so this stand-down
+ # still writes launch_result.json -- the report step reads that file, and a
+ # launcher that exits without one is indistinguishable from a runner that
+ # died, which is the wrong story for a configuration error.
+ owner = username_of(api)
+ if not owner:
+ result["reason"] = "could not determine which Kaggle account this token belongs to"
+ return finish()
+ if args.user and args.user != owner:
+ result["reason"] = (
+ f"the account selected upstream ({args.user}) is not the account this token "
+ f"authenticates as ({owner}); refusing to push, because a kernel pushed "
+ "under one name cannot be deleted under the other"
+ )
+ return finish()
+ args.user = owner
+ _log(f"authenticated as {owner}")
+
# Reclaim anything a previous launcher was killed outright before it
# could delete. Done BEFORE pushing, so the freed session slots are
# available to this run -- Kaggle allows only two GPU sessions at once,
# and an orphan holds one until its ceiling.
- for _slug in sweep_orphans():
+ for _slug in sweep_orphans(owner):
_log(f"reclaimed orphaned kernel {_slug} from a killed launcher")
# AGAIN, now that authentication is paid for and the next thing is a push.
# The check above is not enough on its own: authenticate() reaches the
diff --git a/.github/workflows/kaggle-t4-notebook-ci.yml b/.github/workflows/kaggle-t4-notebook-ci.yml
index de6bc3fe47e..dc2659e69e5 100644
--- a/.github/workflows/kaggle-t4-notebook-ci.yml
+++ b/.github/workflows/kaggle-t4-notebook-ci.yml
@@ -90,9 +90,15 @@
# ---------------------------------------------------------------------
# BUDGET. Read this before changing any trigger.
# ---------------------------------------------------------------------
-# The Kaggle account behind KAGGLE_ACCESS_TOKEN_GH has a WEEKLY accelerator
-# budget (60h at time of writing; Kaggle's documented baseline is 30h and the
-# surplus is a discretionary "floating" allowance that can be withdrawn). This
+# CI holds TWO Kaggle accounts, and each has its own WEEKLY accelerator budget
+# (60h and 30h at time of writing; Kaggle's documented baseline is 30h and any
+# surplus is a discretionary "floating" allowance that can be withdrawn). A run
+# lands on one of them, drawn in proportion to those weekly totals, so the
+# bigger account takes twice the traffic. The totals are read from Kaggle's own
+# quota API each run rather than written down here, so the split follows a plan
+# that changes without this comment going stale. The arithmetic below is per
+# ACCOUNT: two accounts is roughly 1.5x the hours and, because Kaggle's
+# 2-session cap is also per account, twice the concurrency. This
# 50 of those 60 go to CI and the last 10 are held for human use. That 50 is
# SHARED with kaggle-t4-studio-gpu-ci.yml, which is the expensive leg: ~0.75
# GPU-h per launch against this one's ~0.25, on a larger eligible event stream.
@@ -231,11 +237,16 @@
# ---------------------------------------------------------------------
# CREDENTIALS
# ---------------------------------------------------------------------
-# KAGGLE_ACCESS_TOKEN_GH is passed as KAGGLE_API_TOKEN to the gate and the
-# launcher only. It is never echoed, never written to an output or artifact, and
-# never travels to the Kaggle kernel, which receives the notebook and nothing
-# else. Fork pull requests do not receive the secret, and the gate skips cleanly
-# when it is absent.
+# KAGGLE_API_TOKEN and KAGGLE_API_TOKEN_2 hold one Kaggle account each. The
+# GATE is the only place both are visible: it weighs them, picks one, and
+# publishes the account's NUMBER, USERNAME and SECRET NAME -- never a token,
+# which would put a credential in a job output. Every later step receives
+# exactly the chosen one, as KAGGLE_API_TOKEN, selected by indexing the secrets
+# context with that name.
+#
+# Neither is ever echoed, written to an output or artifact, or travels to the
+# Kaggle kernel, which receives the notebook and nothing else. Fork pull
+# requests receive no secret, and the gate skips cleanly when none is present.
#
# WHERE THE TRUST BOUNDARY ACTUALLY IS, since it is easy to draw it in the wrong
# place: a SAME-REPOSITORY pull request does receive the secret, and on
@@ -356,8 +367,8 @@ concurrency:
jobs:
gate:
name: gate
- # Fork pull requests never receive KAGGLE_ACCESS_TOKEN_GH, so this job could
- # only reach "the token is not available to this context", a runner spent to
+ # Fork pull requests never receive the Kaggle secrets, so this job could
+ # only reach "no credential is available to this context", a runner spent to
# learn nothing. The gate handles the absent token cleanly anyway (it skips
# and exits 0); this simply does not start.
if: github.event.pull_request.head.repo.fork != true
@@ -366,6 +377,13 @@ jobs:
outputs:
should_run: ${{ steps.decide.outputs.should_run }}
reason: ${{ steps.decide.outputs.reason }}
+ # WHICH account won the weighted draw. This is the only job that sees
+ # both tokens; everything downstream is handed exactly one, chosen by
+ # these outputs. No token is ever an output -- `matrix` carries the SECRET
+ # NAME, and the GPU job indexes the secrets context with it.
+ account: ${{ steps.decide.outputs.account }}
+ account_user: ${{ steps.decide.outputs.account_user }}
+ matrix: ${{ steps.decide.outputs.matrix }}
steps:
- name: Harden runner (audit)
uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0
@@ -449,7 +467,12 @@ jobs:
- name: Decide
id: decide
env:
- KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_ACCESS_TOKEN_GH }}
+ # BOTH accounts, and this is the only step anywhere that gets both.
+ # The gate reads each one's weekly total to weight the draw and each
+ # one's remaining hours to decide whether it can take the run; every
+ # step after this is handed exactly the one it chose.
+ KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_API_TOKEN }}
+ KAGGLE_API_TOKEN_2: ${{ secrets.KAGGLE_API_TOKEN_2 }}
PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
EVENT_ACTION: ${{ github.event.action }}
EVENT_LABEL: ${{ github.event.label.name }}
@@ -470,6 +493,22 @@ jobs:
needs: gate
if: needs.gate.outputs.should_run == 'true'
runs-on: ubuntu-latest
+ # A ONE-ELEMENT matrix, whose only job is to carry the account the gate
+ # chose into a context that `secrets[...]` can be indexed with.
+ #
+ # It cannot hold the token, and must not: `secrets` is unavailable while the
+ # strategy is expanded, and a token in a job output would be a token in a
+ # place GitHub redacts by pattern rather than by promise. It holds the
+ # SECRET NAME instead.
+ #
+ # The obvious alternative, `${{ cond && secrets.A || secrets.B }}`, is the
+ # trap this shape exists to avoid: a missing or empty first secret makes the
+ # `&&` falsy and the `||` hands over the OTHER account's token, while every
+ # output beside it still names the first account. That run spends one
+ # account's quota, reports the other's, and files kernels the cleanup looks
+ # for in the wrong place. Indexing makes the mismatch unrepresentable.
+ strategy:
+ matrix: ${{ fromJSON(needs.gate.outputs.matrix) }}
# THIS DEADLINE MUST EXCEED THE LAUNCHER'S WORST CASE with headroom, which
# is stronger than "long enough for a normal run". Nothing else stops a
# pushed kernel: killing the runner here takes launch.py's finish() ->
@@ -541,10 +580,13 @@ jobs:
# 32607617804 sat queued about 40 minutes behind notebook run 32607621452.
#
# This only stays safe while each workflow pushes ONE kernel. Two workflows
- # x one kernel is exactly Kaggle's 2-session cap, with no headroom, so the
- # capacity handling on the push path is load bearing rather than defensive.
+ # x one kernel is exactly Kaggle's 2-session cap PER ACCOUNT, with no
+ # headroom, so the capacity handling on the push path is load bearing rather
+ # than defensive.
+ #
concurrency:
- # SLOTTED, because one group cannot express "at most two".
+ # SLOTTED AS WELL AS PER-ACCOUNT, because the two answer different
+ # halves of the same cap and neither is redundant.
#
# A Kaggle account allows 2 concurrent GPU sessions and this job takes
# one, so the cap has to be 2 and GitHub concurrency only offers 1 per
@@ -552,14 +594,20 @@ jobs:
# serialised: only ONE run may be PENDING in a group, so a second queued
# run CANCELS the first rather than queueing behind it. That is what
# killed run 32674255736, and it makes an A/B impossible to run at all.
+ # Two slots, chosen explicitly by the dispatcher, is that mechanism. It is
+ # bounded by construction (the input offers 1 or 2 and nothing else), so
+ # one account can never be asked for a third session, and two dispatches
+ # naming different slots genuinely run side by side. Everything that is
+ # not a dispatch has an empty input and lands in slot 1.
+ #
+ # The ACCOUNT half is what makes a second account worth having. Without it
+ # every run queues behind every other run whichever account it would
+ # spend, and the extra hours sit unused. With both, the account gets its
+ # own pair of slots: 2 accounts x 2 slots, still Kaggle's 2 per account.
#
- # Two slots, chosen explicitly by the dispatcher, is the whole mechanism.
- # It is bounded by construction (the input offers 1 or 2 and nothing
- # else), so the account can never be asked for a third session, and two
- # dispatches that name different slots genuinely run side by side.
- # Everything that is not a dispatch has an empty input and lands in slot
- # 1, which is exactly the old behaviour for pushes and pull requests.
- group: kaggle-t4-account-notebook-${{ inputs.slot || '1' }}
+ # `needs` is available here and `secrets` is not, which is the other
+ # reason the gate publishes an account NUMBER rather than a credential.
+ group: kaggle-t4-account-${{ needs.gate.outputs.account }}-notebook-${{ inputs.slot || '1' }}
cancel-in-progress: false
steps:
# FIRST, ahead of even the audit hardening, and it is an `echo` so there
@@ -965,11 +1013,16 @@ jobs:
id: recheck
if: steps.ref.outputs.stand_down != 'true' && steps.pins.outputs.stand_down != 'true' && steps.stepcount.outputs.stand_down != 'true'
env:
- KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_ACCESS_TOKEN_GH }}
+ # ONE account: the one the gate chose. `--account-env` narrows the
+ # recheck to it, so this step cannot quietly clear a DIFFERENT account
+ # from the one whose slot this job is holding and whose kernels it is
+ # about to push.
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
run: |
python .github/scripts/kaggle_t4_ci/gate.py \
--force true \
--soft-fail \
+ --account-env KAGGLE_API_TOKEN \
--kernels 1 \
--budget-hours 4 \
--reserve-hours 20
@@ -1003,12 +1056,12 @@ jobs:
id: launch
if: steps.recheck.outputs.should_run == 'true'
env:
- KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_ACCESS_TOKEN_GH }}
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
JOB_TIMEOUT_MINUTES: 260
run: |
python .github/scripts/kaggle_t4_ci/launch.py \
${{ steps.build.outputs.notebooks }} \
- --user danielhanchen \
+ --user '${{ matrix.kaggle_user }}' \
--outdir kaggle_evidence \
--expect '${{ steps.build.outputs.payloads }}' \
--kernel-timeout-sec 3600 \
diff --git a/.github/workflows/kaggle-t4-studio-gpu-ci.yml b/.github/workflows/kaggle-t4-studio-gpu-ci.yml
index d9628e7c08b..64bf7301409 100644
--- a/.github/workflows/kaggle-t4-studio-gpu-ci.yml
+++ b/.github/workflows/kaggle-t4-studio-gpu-ci.yml
@@ -41,11 +41,16 @@
# ---------------------------------------------------------------------
# BUDGET. Read this before changing any trigger.
# ---------------------------------------------------------------------
-# The Kaggle account behind KAGGLE_ACCESS_TOKEN_GH has a WEEKLY accelerator
-# budget (60h at time of writing) shared with every other use of that
-# account, and with the notebook leg in kaggle-t4-notebook-ci.yml. CI is
-# allowed up to 50 of those 60; the last 10 are held for human use and are
-# what `--reserve-hours 10` protects.
+# CI holds TWO Kaggle accounts, with a WEEKLY accelerator budget each (60h
+# and 30h at time of writing), shared with every other use of those accounts
+# and with the notebook leg in kaggle-t4-notebook-ci.yml. A run is drawn onto
+# one of them in proportion to those totals, read from Kaggle's quota API each
+# run rather than written down here.
+#
+# `--reserve-hours 10` is what the last sixth is held back by, and it SCALES:
+# ten hours out of a 60h plan is the same fraction as five out of a 30h one, so
+# the smaller account is not quietly made the stricter one. See scaled_reserve
+# in .github/scripts/kaggle_t4_ci/gate.py.
#
# Unsloth is sampled HARDER than the notebook leg because Unsloth changes land
# far more often. That comes out as MORE launches, far more GPU-hours, a
@@ -158,11 +163,15 @@
# ---------------------------------------------------------------------
# CREDENTIALS
# ---------------------------------------------------------------------
-# KAGGLE_ACCESS_TOKEN_GH is passed as KAGGLE_API_TOKEN to the gate and the
-# launcher only. It is never echoed, never written to an output or an
-# artifact, and never travels to the Kaggle kernel: the kernel receives the
-# notebook and nothing else. Fork pull requests do not receive the secret and
-# the gate skips cleanly when it is absent.
+# KAGGLE_API_TOKEN and KAGGLE_API_TOKEN_2 hold one account each. The GATE is
+# the only step that sees both: it weighs them, picks one, and publishes that
+# account's NUMBER, USERNAME and SECRET NAME. No token is ever a job output.
+# Every later step gets exactly the chosen one, as KAGGLE_API_TOKEN, by
+# indexing the secrets context with that name.
+#
+# Neither is echoed, written to an output or an artifact, or travels to the
+# Kaggle kernel: the kernel receives the notebook and nothing else. Fork pull
+# requests receive no secret and the gate skips cleanly when none is present.
#
# The payload mints its own Unsloth bootstrap password inside the kernel and
# scrubs it out of every log that leaves the machine, because Unsloth's
@@ -248,8 +257,8 @@ concurrency:
jobs:
gate:
name: gate
- # Fork pull requests never receive KAGGLE_ACCESS_TOKEN_GH, so this job
- # could only reach "the token is not available to this context" -- a
+ # Fork pull requests never receive the Kaggle secrets, so this job
+ # could only reach "no credential is available to this context" -- a
# runner spent to learn nothing.
#
# The `labeled` clause is what keeps the opt-in label from turning into a
@@ -271,6 +280,12 @@ jobs:
outputs:
should_run: ${{ steps.decide.outputs.should_run }}
reason: ${{ steps.decide.outputs.reason }}
+ # The account the weighted draw landed on. Only this job sees both
+ # tokens; `matrix` carries the SECRET NAME so the GPU job can index the
+ # secrets context, never a credential.
+ account: ${{ steps.decide.outputs.account }}
+ account_user: ${{ steps.decide.outputs.account_user }}
+ matrix: ${{ steps.decide.outputs.matrix }}
steps:
- name: Harden runner (audit)
uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0
@@ -307,7 +322,10 @@ jobs:
- name: Decide
id: decide
env:
- KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_ACCESS_TOKEN_GH }}
+ # BOTH accounts, and the only step here that sees both: the gate
+ # weighs their weekly totals, picks one, and hands its NAME down.
+ KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_API_TOKEN }}
+ KAGGLE_API_TOKEN_2: ${{ secrets.KAGGLE_API_TOKEN_2 }}
run: |
python .github/scripts/kaggle_t4_ci/gate.py \
--percent 5 \
@@ -323,6 +341,13 @@ jobs:
needs: gate
if: needs.gate.outputs.should_run == 'true'
runs-on: ubuntu-latest
+ # One element, carrying the chosen account into a context `secrets[...]`
+ # can be indexed with. It holds the SECRET NAME, never the token: `secrets`
+ # is not available while the strategy expands, and a `cond && A || B`
+ # ternary silently yields the OTHER account's token when the first secret is
+ # empty, while the metadata beside it still names the first.
+ strategy:
+ matrix: ${{ fromJSON(needs.gate.outputs.matrix) }}
# 120 min: the kernel ceiling is 70 and collection adds a few, so this
# only fires if the launcher itself wedged.
timeout-minutes: 120
@@ -340,8 +365,13 @@ jobs:
# arithmetic has no slack -- one kernel here plus one there is exactly
# Kaggle's 2-session cap -- so anything that makes either side push a second
# kernel has to put these two back in one group.
+ # KEYED ON THE ACCOUNT. One string for the whole workflow meant every
+ # Studio run queued behind every other one whichever account it would
+ # spend, so a second account added no capacity at all. Per account it is
+ # one Studio kernel each, beside one notebook kernel each: four sessions
+ # over two accounts, still Kaggle's two per account.
concurrency:
- group: kaggle-t4-account-studio
+ group: kaggle-t4-account-${{ needs.gate.outputs.account }}-studio
cancel-in-progress: false
steps:
- name: Harden runner (audit)
@@ -398,6 +428,40 @@ jobs:
--payload-args "$ARGS" \
--per-run-timeout 3900
+ # WITH THE ACCOUNT SLOT IN HAND, and immediately before the push. The
+ # notebook leg has always done this and this one never has; two accounts
+ # is what makes the gap matter. A gate answer can be a whole run stale by
+ # the time the concurrency group lets this job start -- the repo queue has
+ # been measured 293 deep -- and in that window the chosen account can have
+ # spent its hours or taken its second session on another branch.
+ #
+ # --account-env narrows the recheck to the ONE account the gate chose, so
+ # it can never clear a different account from the one this job holds a
+ # slot for. --soft-fail because the run is already past the gate and only
+ # re-asking: exhaustion discovered here is a stand-down, not a red, and
+ # the step below says so.
+ - name: Recheck the Kaggle account
+ id: recheck
+ env:
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
+ run: |
+ python .github/scripts/kaggle_t4_ci/gate.py \
+ --force true \
+ --soft-fail \
+ --account-env KAGGLE_API_TOKEN \
+ --kernels 1 \
+ --budget-hours 2 \
+ --reserve-hours 10
+
+ # Through the environment rather than an inline expression, so nothing the
+ # Kaggle API put in that string reaches a shell.
+ - name: Report the stale approval
+ if: steps.recheck.outputs.should_run != 'true'
+ env:
+ REASON: ${{ steps.recheck.outputs.reason }}
+ run: |
+ echo "::warning title=Stood down at the second check::${REASON}"
+
# kernel-timeout-sec is the load-bearing budget control: Kaggle itself
# kills the session at that point. A runner that is cancelled cannot
# clean up after itself, so the ceiling has to be enforced on Kaggle's
@@ -408,12 +472,16 @@ jobs:
# needed no modification.
- name: Launch on Kaggle and collect
id: launch
+ # GATED ON THE RECHECK, or the recheck is decorative: a step that
+ # measures the account and cannot stop the push is a log line, not a
+ # control. This is the same shape the notebook leg uses.
+ if: steps.recheck.outputs.should_run == 'true'
env:
- KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_ACCESS_TOKEN_GH }}
+ KAGGLE_API_TOKEN: ${{ secrets[matrix.secret_name] }}
run: |
python .github/scripts/kaggle_t4_ci/launch.py \
--notebook kernel.ipynb \
- --user danielhanchen \
+ --user '${{ matrix.kaggle_user }}' \
--outdir kaggle_evidence \
--expect 1 \
--kernel-timeout-sec 4200 \
diff --git a/tests/kaggle/conftest.py b/tests/kaggle/conftest.py
new file mode 100644
index 00000000000..8371660867a
--- /dev/null
+++ b/tests/kaggle/conftest.py
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""No test in this directory may see a real Kaggle credential.
+
+Found the hard way. `gate.py` now considers two accounts, and a developer box
+with `KAGGLE_API_TOKEN_2` exported turned the "a fork gets no secret" test into a
+LIVE run: the gate authenticated to a real account, read that week's actual
+remaining hours over the network, and answered `should_run=true`. The test failed
+for the right-looking reason and would have PASSED had the quota been low, which
+is the worse version -- a unit test whose verdict depends on somebody's usage.
+
+Every credential env var is therefore cleared before each test, and a test that
+wants one sets it explicitly afterwards. Autouse, because the hazard is in the
+tests that never think about credentials at all.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+CI_DIR = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "kaggle_t4_ci"
+sys.path.insert(0, str(CI_DIR))
+
+# Read from the gate rather than restated, so an account added there is covered
+# here without anyone remembering to.
+try:
+ from gate import DEFAULT_ACCOUNT_ENVS
+except Exception: # noqa: BLE001 - the suite has its own import guards
+ DEFAULT_ACCOUNT_ENVS = ("KAGGLE_API_TOKEN", "KAGGLE_API_TOKEN_2")
+
+# The client reads these too, and a stray one authenticates just as well as the
+# token does.
+_OTHER_CREDENTIAL_ENVS = ("KAGGLE_KEY", "KAGGLE_USERNAME", "KAGGLE_ACCESS_TOKEN_GH")
+
+
[email protected](autouse = True)
+def _no_ambient_kaggle_credentials(monkeypatch):
+ for name in (*DEFAULT_ACCOUNT_ENVS, *_OTHER_CREDENTIAL_ENVS):
+ monkeypatch.delenv(name, raising = False)
diff --git a/tests/kaggle/test_launch_cleanup.py b/tests/kaggle/test_launch_cleanup.py
index fbb091fa516..d106fa3d176 100644
--- a/tests/kaggle/test_launch_cleanup.py
+++ b/tests/kaggle/test_launch_cleanup.py
@@ -36,6 +36,26 @@ sys.path.insert(0, str(CI_DIR))
import launch # noqa: E402
+class _StubKaggleApi:
+ """A client that can say WHICH account it is, because the real one can.
+
+ `launch.py` reads the owner off the authenticated client and refuses to push
+ when it cannot: a kernel id is `<owner>/<slug>`, CI holds more than one
+ account, and a kernel pushed under the wrong name cannot be deleted under
+ the other. A bare `object()` models a client that never authenticated, which
+ is a different test from the ones below.
+ """
+
+ CONFIG_NAME_USER = "username"
+
+ def __init__(self, username = "me"):
+ self.config_values = {self.CONFIG_NAME_USER: username}
+
+
+def _stub_api(*_args, **_kwargs):
+ return _StubKaggleApi()
+
+
def _fake_kaggle(bin_dir: Path, record: Path) -> None:
"""A `kaggle` on PATH that only records what it was asked to delete."""
bin_dir.mkdir(parents = True, exist_ok = True)
@@ -248,7 +268,7 @@ def _run_main(
# The retry backoffs, not the retries: every attempt still runs.
monkeypatch.setattr(launch, "PUSH_BACKOFF_SEC", 0)
monkeypatch.setattr(launch, "DELETE_BACKOFF_SEC", 0)
- monkeypatch.setattr(launch, "_api", lambda: object())
+ monkeypatch.setattr(launch, "_api", _stub_api)
monkeypatch.setattr(launch, "wait", lambda *a, **kw: "COMPLETE")
monkeypatch.setattr(
launch,
@@ -463,8 +483,11 @@ def _waiting_launcher(outdir: Path) -> str:
return "\n".join(
[
"",
- " launch._api = lambda: object()",
- " launch.sweep_orphans = lambda: []",
+ " class _Api:",
+ " CONFIG_NAME_USER = 'username'",
+ " config_values = {'username': 'me'}",
+ " launch._api = lambda *a, **k: _Api()",
+ " launch.sweep_orphans = lambda *a, **k: []",
" def _push(notebook, user, kernel_timeout_sec,",
" accelerator='NvidiaTeslaT4', attempted=None):",
" attempted = [] if attempted is None else attempted",
@@ -1042,8 +1065,11 @@ def test_a_kernel_pushed_before_the_signal_is_still_deleted(tmp_path):
body = "\n".join(
[
"",
- " launch._api = lambda: object()",
- " launch.sweep_orphans = lambda: []",
+ " class _Api:",
+ " CONFIG_NAME_USER = 'username'",
+ " config_values = {'username': 'me'}",
+ " launch._api = lambda *a, **k: _Api()",
+ " launch.sweep_orphans = lambda *a, **k: []",
" calls = []",
" def _push(notebook, user, kernel_timeout_sec,",
" accelerator='NvidiaTeslaT4', attempted=None):",
diff --git a/tests/kaggle/test_studio_gpu_harness.py b/tests/kaggle/test_studio_gpu_harness.py
index 2b81a86afa6..d2671fdae31 100644
--- a/tests/kaggle/test_studio_gpu_harness.py
+++ b/tests/kaggle/test_studio_gpu_harness.py
@@ -2250,8 +2250,16 @@ def test_the_gate_is_told_how_many_kernels_this_leg_actually_pushes():
import re
text = WORKFLOW.read_text(encoding = "utf-8")
- gate = text.split("gate.py", 1)[1].split("\n\n", 1)[0]
- assert re.search(r"^\s+--kernels 1 \\\s*$", gate, re.MULTILINE), gate
+ # EVERY invocation, found by the command rather than by splitting on the
+ # first literal "gate.py" in the file: prose above the jobs mentions the
+ # script by path, and there are now two calls -- the gate and the recheck
+ # that re-asks with the account slot in hand. Both push one kernel, so both
+ # have to say so, and a recheck left at the default of two would stand the
+ # job down for a slot it does not need.
+ invocations = re.findall(r"kaggle_t4_ci/gate\.py \\\n.*?(?=\n\s*\n|\Z)", text, re.DOTALL)
+ assert invocations, "the workflow never runs the gate"
+ for invocation in invocations:
+ assert re.search(r"^\s+--kernels 1 \\?\s*$", invocation, re.MULTILINE), invocation
assert "--expect 1" in text
diff --git a/tests/kaggle/test_t4_ci_transport.py b/tests/kaggle/test_t4_ci_transport.py
index 62c71a9531b..5af9adc3820 100644
--- a/tests/kaggle/test_t4_ci_transport.py
+++ b/tests/kaggle/test_t4_ci_transport.py
@@ -39,6 +39,26 @@ import launch # noqa: E402
from legs import KERNELS, LEGS # noqa: E402
+class _StubKaggleApi:
+ """A client that can say WHICH account it is, because the real one can.
+
+ `launch.py` reads the owner off the authenticated client and refuses to push
+ when it cannot: a kernel id is `<owner>/<slug>`, CI holds more than one
+ account, and a kernel pushed under the wrong name cannot be deleted under
+ the other. A bare `object()` models a client that never authenticated, which
+ is a different test from the ones below.
+ """
+
+ CONFIG_NAME_USER = "username"
+
+ def __init__(self, username = "someuser"):
+ self.config_values = {self.CONFIG_NAME_USER: username}
+
+
+def _stub_api(*_args, **_kwargs):
+ return _StubKaggleApi()
+
+
# ------------------------------------------------------------------ driver
@@ -1137,9 +1157,9 @@ def _drive_main(
monkeypatch.setattr(launch.time, "time", lambda: clock["t"])
monkeypatch.setattr(launch.time, "sleep", lambda _s: None)
- def fake_api():
+ def fake_api(*_args, **_kwargs):
clock["t"] += api_seconds
- return object()
+ return _StubKaggleApi()
monkeypatch.setattr(launch, "_api", fake_api)
@@ -1477,7 +1497,7 @@ def test_a_push_that_times_out_does_not_abandon_the_kernel_already_accepted(monk
monkeypatch.setattr(launch.subprocess, "run", fake_run)
monkeypatch.setattr(launch.time, "sleep", lambda _s: None)
- monkeypatch.setattr(launch, "_api", lambda: object())
+ monkeypatch.setattr(launch, "_api", _stub_api)
monkeypatch.setattr(launch, "wait", lambda api, slug, poll_every, max_wait: "COMPLETE")
monkeypatch.setattr(
launch, "fetch_evidence", lambda slug, outdir, **kw: {"notebooks": [], "log": None}
@@ -1563,7 +1583,7 @@ def test_a_push_that_raises_outside_the_timeout_still_gives_up_its_slug(
monkeypatch.setattr(launch.subprocess, "run", fake_run)
monkeypatch.setattr(launch.time, "sleep", lambda _s: None)
- monkeypatch.setattr(launch, "_api", lambda: object())
+ monkeypatch.setattr(launch, "_api", _stub_api)
monkeypatch.setattr(launch, "wait", lambda api, slug, poll_every, max_wait: "COMPLETE")
monkeypatch.delenv("GITHUB_OUTPUT", raising = False)
(tmp_path / "k0.ipynb").write_text("{}", encoding = "utf-8")
@@ -1642,7 +1662,7 @@ def test_an_abort_anywhere_in_the_launcher_still_deletes_what_it_pushed(monkeypa
deleted.append(cmd[3])
return types.SimpleNamespace(returncode = 0, stdout = "", stderr = "")
- monkeypatch.setattr(launch, "_api", lambda: object())
+ monkeypatch.setattr(launch, "_api", _stub_api)
monkeypatch.setattr(launch, "push", _accepting_push("someuser/unsloth-t4-ci-abcd"))
monkeypatch.setattr(launch, "wait", lambda api, slug, poll_every, max_wait: "COMPLETE")
monkeypatch.setattr(
@@ -1683,7 +1703,7 @@ def _drive_one_kernel(monkeypatch, tmp_path, fake_run):
Everything but the delete calls is stubbed, so what comes back reads the
release path and nothing else.
"""
- monkeypatch.setattr(launch, "_api", lambda: object())
+ monkeypatch.setattr(launch, "_api", _stub_api)
monkeypatch.setattr(launch, "push", _accepting_push("someuser/unsloth-t4-ci-abcd"))
monkeypatch.setattr(launch, "wait", lambda api, slug, poll_every, max_wait: "COMPLETE")
monkeypatch.setattr(
@@ -2322,7 +2342,7 @@ def test_main_bounds_the_whole_evidence_phase_it_is_budgeted_for(monkeypatch, tm
"extract_reports",
lambda outdir: [{"label": "control", "model": "m", "passed": True}],
)
- monkeypatch.setattr(launch, "_api", lambda: object())
+ monkeypatch.setattr(launch, "_api", _stub_api)
monkeypatch.setattr(launch, "delete_kernel", lambda slug: True)
monkeypatch.delenv("GITHUB_OUTPUT", raising = False)
monkeypatch.setattr(
diff --git a/tests/kaggle/test_t4_smoke_harness.py b/tests/kaggle/test_t4_smoke_harness.py
index beea60737e8..d2c24a20d30 100644
--- a/tests/kaggle/test_t4_smoke_harness.py
+++ b/tests/kaggle/test_t4_smoke_harness.py
@@ -545,7 +545,9 @@ def test_a_gate_error_is_a_skip_not_a_failure(monkeypatch, tmp_path):
import gate
monkeypatch.setenv("KAGGLE_API_TOKEN", "not-a-real-token")
- monkeypatch.setattr(gate, "kaggle_client", lambda: (_ for _ in ()).throw(RuntimeError("boom")))
+ monkeypatch.setattr(
+ gate, "kaggle_client", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))
+ )
code, outputs = _run_gate(monkeypatch, tmp_path, "--force", "true")
assert code == 0
assert outputs["should_run"] == "false"
@@ -568,7 +570,7 @@ def test_the_gate_bounds_its_network_calls_before_it_makes_one(monkeypatch, tmp_
seen = {}
- def _client():
+ def _client(*_args, **_kwargs):
seen["timeout"] = socket.getdefaulttimeout()
raise RuntimeError("boom")
@@ -663,7 +665,11 @@ def _run_gate_against(
answer = _idle_account() if survey is None else survey
monkeypatch.setenv("KAGGLE_API_TOKEN", "not-a-real-token")
- monkeypatch.setattr(gate, "kaggle_client", lambda: object())
+ monkeypatch.setattr(gate, "kaggle_client", lambda *a, **k: object())
+ # The account a token belongs to. Stubbed rather than left out: the gate
+ # REFUSES an account it cannot name, because the launcher must push under
+ # that name, so a nameless client is a stand-down rather than a default.
+ monkeypatch.setattr(gate, "client_username", lambda api: "danielhanchen")
monkeypatch.setattr(gate, "remaining_gpu_hours", lambda api: dict(quota))
monkeypatch.setattr(
gate, "survey_kernels", answer if callable(answer) else (lambda api: answer)
@@ -776,7 +782,9 @@ def test_soft_failure_is_asked_for_rather_than_assumed(monkeypatch, tmp_path):
# And --no-soft-fail still means what it always did: an error in the gate
# itself becomes a failure rather than a skip.
monkeypatch.setenv("KAGGLE_API_TOKEN", "not-a-real-token")
- monkeypatch.setattr(gate, "kaggle_client", lambda: (_ for _ in ()).throw(RuntimeError("boom")))
+ monkeypatch.setattr(
+ gate, "kaggle_client", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))
+ )
hard, _ = _run_gate(monkeypatch, tmp_path / "hard", "--force", "true", "--no-soft-fail")
assert hard == 1
soft, outputs = _run_gate(monkeypatch, tmp_path / "soft", "--force", "true", "--soft-fail")
@@ -1620,6 +1628,7 @@ def test_built_kernel_is_valid_notebook_json_with_gpu_requested(tmp_path):
blob = json.dumps(nb)
for forbidden in (
"KAGGLE_API_TOKEN",
+ "KAGGLE_API_TOKEN_2",
"KAGGLE_KEY",
"KAGGLE_USERNAME",
"KAGGLE_ACCESS_TOKEN_GH",
diff --git a/tests/kaggle/test_two_account_selection.py b/tests/kaggle/test_two_account_selection.py
new file mode 100644
index 00000000000..ce337a9dcda
--- /dev/null
+++ b/tests/kaggle/test_two_account_selection.py
@@ -0,0 +1,454 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Two Kaggle accounts: the draw, the handover, and the ways this goes quiet.
+
+The failure this whole file is written against is not an exception. It is a
+GREEN run that spent the wrong account, or spent nothing at all and said so in a
+way nobody reads. That is not hypothetical here: the workflows referenced a
+secret that had been deleted, `gate.py` answers a missing credential with a skip
+that exits 0, and both Kaggle workflows were therefore a silent no-op on main
+with every check passing. The first test below is the one that would have caught
+it, so it is written first.
+
+Everything here runs on CPU with no Kaggle quota spent: the client is a stub, so
+the only thing that ever reaches the network in these paths is not reached.
+"""
+
+from __future__ import annotations
+
+import collections
+import json
+import re
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+CI_DIR = REPO_ROOT / ".github" / "scripts" / "kaggle_t4_ci"
+WORKFLOWS = REPO_ROOT / ".github" / "workflows"
+NOTEBOOK_WF = WORKFLOWS / "kaggle-t4-notebook-ci.yml"
+STUDIO_WF = WORKFLOWS / "kaggle-t4-studio-gpu-ci.yml"
+
+sys.path.insert(0, str(CI_DIR))
+
+import gate # noqa: E402
+import launch # noqa: E402
+
+
+def _wf(path: Path) -> dict:
+ return yaml.safe_load(path.read_text(encoding = "utf-8"))
+
+
+def _steps(workflow: dict) -> list[tuple[str, str, dict]]:
+ out = []
+ for job_name, job in workflow["jobs"].items():
+ for step in job.get("steps", []) or []:
+ out.append((job_name, step.get("name", ""), step))
+ return out
+
+
+# --------------------------------------------------------------- the secrets
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_no_workflow_names_a_secret_that_does_not_exist(path):
+ """THE GUARD THIS FILE EXISTS FOR, and it is written from a real outage.
+
+ `KAGGLE_ACCESS_TOKEN_GH` was deleted from the repository. Both workflows
+ still named it, so every step got an empty string, and `gate.py` reads an
+ absent credential as "expected on a fork" and skips with exit 0. Two GPU
+ workflows became a no-op and every check stayed green.
+
+ So the set of Kaggle secrets a workflow may reference is CLOSED, and any
+ name outside it fails here rather than on the next quiet Sunday.
+ """
+ referenced = set(re.findall(r"secrets\.([A-Z0-9_]+)", path.read_text(encoding = "utf-8")))
+ kaggle = {s for s in referenced if "KAGGLE" in s}
+ assert kaggle == set(gate.DEFAULT_ACCOUNT_ENVS), (
+ f"{path.name} names Kaggle secrets {sorted(kaggle)}, but the accounts that "
+ f"exist are {sorted(gate.DEFAULT_ACCOUNT_ENVS)}. A name that is not a real "
+ "secret resolves to an empty string and the gate skips green."
+ )
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_every_step_that_runs_a_kaggle_script_is_given_a_token(path):
+ """Derived from the workflow, not from a list, so a step added later cannot
+ quietly run credential-less and report the skip as a normal outcome."""
+ missing = []
+ for job_name, step_name, step in _steps(_wf(path)):
+ body = step.get("run") or ""
+ if "kaggle_t4_ci/gate.py" not in body and "kaggle_t4_ci/launch.py" not in body:
+ continue
+ env = step.get("env") or {}
+ if "KAGGLE_API_TOKEN" not in env:
+ missing.append(f"{job_name}/{step_name}")
+ assert missing == [], f"these steps run a Kaggle script with no token: {missing}"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_only_the_gate_sees_both_accounts(path):
+ """One account per step, everywhere except the one step that chooses.
+
+ A later step holding both tokens could authenticate as either, which is the
+ state the account output exists to make impossible.
+ """
+ for job_name, step_name, step in _steps(_wf(path)):
+ env = step.get("env") or {}
+ tokens = sorted(k for k in env if k.startswith("KAGGLE_API_TOKEN"))
+ if not tokens:
+ continue
+ if step.get("id") == "decide":
+ assert tokens == sorted(
+ gate.DEFAULT_ACCOUNT_ENVS
+ ), f"the gate must weigh every account, got {tokens}"
+ else:
+ assert tokens == [
+ "KAGGLE_API_TOKEN"
+ ], f"{job_name}/{step_name} sees {tokens}; only the gate may see more than one"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_the_chosen_token_is_INDEXED_and_never_a_ternary(path):
+ """`${{ cond && secrets.A || secrets.B }}` is the shape this must not use.
+
+ An empty or missing first secret makes the `&&` falsy, the `||` hands over
+ the OTHER account's token, and every output beside it still names the first
+ account: the run spends one account and reports another, and the cleanup
+ then looks for its kernels under a username that does not own them.
+
+ Indexing the secrets context with a name carried in the matrix cannot
+ express that state at all, which is why it is required rather than
+ preferred.
+ """
+ for job_name, step_name, step in _steps(_wf(path)):
+ env = step.get("env") or {}
+ expr = env.get("KAGGLE_API_TOKEN", "")
+ if step.get("id") == "decide":
+ continue
+ if not expr:
+ continue
+ assert "&&" not in expr and "||" not in expr, (
+ f"{job_name}/{step_name} selects its token with a ternary ({expr}), which "
+ "silently falls through to the other account when the first secret is empty"
+ )
+ assert (
+ "secrets[matrix." in expr
+ ), f"{job_name}/{step_name} does not index the secrets context: {expr}"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_no_token_is_ever_a_job_output(path):
+ """The matrix carries a secret NAME. A token in an output is a credential in
+ a place GitHub redacts by pattern rather than by promise."""
+ for job_name, job in _wf(path)["jobs"].items():
+ for key, value in (job.get("outputs") or {}).items():
+ assert "secrets." not in str(value) and "secrets[" not in str(
+ value
+ ), f"{job_name}.outputs.{key} publishes a secret: {value}"
+
+
+# ----------------------------------------------------------- the concurrency
+
+
+ "path,suffix",
+ ((NOTEBOOK_WF, "notebook"), (STUDIO_WF, "studio")),
+ ids = ("notebook", "studio"),
+)
+def test_the_concurrency_group_is_keyed_on_the_account(path, suffix):
+ """Kaggle's 2-session cap is per ACCOUNT, so the lock must be too.
+
+ One group for the whole workflow means every run queues behind every other
+ run whichever account it would spend, and a second account adds no capacity
+ at all -- the run is green, the hours exist, and nothing uses them.
+ """
+ groups = [
+ (job.get("concurrency") or {}).get("group")
+ for job in _wf(path)["jobs"].values()
+ if isinstance(job.get("concurrency"), dict)
+ ]
+ account_groups = [g for g in groups if g and suffix in g]
+ assert account_groups, f"no per-account {suffix} group found in {path.name}"
+ for group in account_groups:
+ assert (
+ "needs.gate.outputs.account" in group
+ ), f"{group!r} does not vary by account, so two accounts share one lock"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_the_gpu_job_takes_the_account_through_a_one_element_matrix(path):
+ for job_name, job in _wf(path)["jobs"].items():
+ if job_name == "gate":
+ continue
+ matrix = (job.get("strategy") or {}).get("matrix")
+ assert matrix and "needs.gate.outputs.matrix" in str(
+ matrix
+ ), f"{job_name} does not receive the gate's account matrix: {matrix!r}"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_no_kaggle_username_is_hardcoded_on_the_launch_path(path):
+ """A kernel id is `<owner>/<slug>`. A literal owner belongs to whichever
+ account happened to be first when it was typed, so the other account cannot
+ push under it -- and, worse, cannot DELETE under it, which turns a leak into
+ a log line indistinguishable from a kernel that was already gone."""
+ for job_name, step_name, step in _steps(_wf(path)):
+ body = step.get("run") or ""
+ if "launch.py" not in body:
+ continue
+ # To end of line, not the next token: the value is quoted, so `\S+`
+ # captures `'${{` and reports a correct workflow as hardcoded.
+ for match in re.findall(r"--user\s+(.+)", body):
+ assert (
+ "matrix.kaggle_user" in match
+ ), f"{job_name}/{step_name} pushes under a hardcoded owner {match!r}"
+
+
[email protected]("path", (NOTEBOOK_WF, STUDIO_WF), ids = ("notebook", "studio"))
+def test_the_recheck_can_actually_stop_the_push(path):
+ """Both workflows re-ask with the account slot in hand, and in BOTH the push
+ is gated on that answer. A measurement that cannot stop anything is a log
+ line, and the Studio leg had exactly that gap: no recheck at all, so a gate
+ answer a whole queue-wait old was the last word before a session was spent.
+ """
+ steps = _steps(_wf(path))
+ recheck = [s for _, _, s in steps if s.get("id") == "recheck"]
+ assert recheck, f"{path.name} never re-asks the gate with the slot in hand"
+ assert "--account-env KAGGLE_API_TOKEN" in (recheck[0].get("run") or ""), (
+ "the recheck must be narrowed to the ONE account the gate chose, or it can "
+ "clear an account this job is not holding a slot for"
+ )
+ launched = [s for _, _, s in steps if "launch.py" in (s.get("run") or "")]
+ assert launched, f"{path.name} never launches"
+ for step in launched:
+ assert "steps.recheck.outputs.should_run == 'true'" in (
+ step.get("if") or ""
+ ), "the push does not depend on the recheck, so the recheck decides nothing"
+
+
+# ------------------------------------------------------------------ the draw
+
+
+def test_the_split_follows_the_weekly_hours():
+ """60 and 30 must come out 2:1, and the weights are the accounts' own
+ totals rather than a number written here."""
+ counts = collections.Counter(
+ gate.weighted_pick(str(i), {"1": 60.0, "2": 30.0})[0] for i in range(60000)
+ )
+ share = counts["1"] / sum(counts.values())
+ assert 0.65 < share < 0.685, f"account 1 took {share:.4f} of the traffic, wanted ~0.667"
+
+
+def test_a_rerun_returns_to_the_same_account(monkeypatch):
+ """Keyed on the run id ALONE. A re-run of a run whose kernels are still in
+ flight must go back to the account that holds them: the other account cannot
+ delete them, so a rerolled attempt strands the first attempt's session.
+
+ The ATTEMPT is varied here rather than just calling twice. Calling twice
+ only proves the function is deterministic, which it would be even if it read
+ `GITHUB_RUN_ATTEMPT` -- that value does not change inside one process, so a
+ mutation adding it survived the earlier version of this test.
+ """
+ weights = {"1": 60.0, "2": 30.0}
+ for run_id in ("1", "17", "912837", "40000000001"):
+ picks = set()
+ for attempt in ("1", "2", "3", "17"):
+ monkeypatch.setenv("GITHUB_RUN_ATTEMPT", attempt)
+ picks.add(gate.weighted_pick(run_id, weights)[0])
+ assert len(picks) == 1, (
+ f"run {run_id} lands on {picks} across attempts; a re-run would push to an "
+ "account that is not holding the previous attempt's kernels"
+ )
+
+
+def test_the_account_draw_is_salted_apart_from_the_sampling_draw():
+ """Two decisions off one run id, so they are salted apart.
+
+ STATED PLAINLY BECAUSE IT LIMITS THE CLAIM: the statistical version of this
+ test does not work, and it was tried. Removing the salt leaves `sampled_in`
+ reading `digest % 100` and this draw reading `digest % 1_000_000`, and those
+ are independent enough that the measured share of account 1 among sampled-in
+ runs moved from a 0.0039 gap to a 0.0059 one over 200k ids -- both inside
+ noise. A test asserting independence therefore CANNOT fail on the mutation
+ it exists for, which is a test that only looks like coverage.
+
+ So this asserts the derivation instead: the two draws must not hash the same
+ string. That is checkable, and it keeps the property from being removed by
+ someone who has not measured what removing it does.
+ """
+ source = (CI_DIR / "gate.py").read_text(encoding = "utf-8")
+ picked = source.split("def weighted_pick", 1)[1].split("\ndef ", 1)[0]
+ assert (
+ 'sha256(("account:" + run_id)' in picked
+ ), "the account draw hashes the bare run id, which is what sampled_in hashes"
+
+
+def test_an_account_with_no_readable_quota_gets_no_weight_but_keeps_its_turn():
+ """Unknown is not zero-sized and it is not disqualified either: the account
+ loses its share of the traffic, because a share is what its plan says and we
+ did not get to hear it, and keeps its place as a fallback."""
+ chosen, _ = gate.weighted_pick("5", {"1": 60.0})
+ assert chosen == "1"
+ chosen, _ = gate.weighted_pick("5", {})
+ assert chosen == ""
+
+
+# --------------------------------------------------------------- the reserve
+
+
+def test_the_reserve_is_a_fraction_of_the_plan_not_a_flat_number():
+ """20h held out of a 30h account is two thirds of it against one third of a
+ 60h one, which silently makes the SMALLER account the stricter one -- on top
+ of it already taking less traffic by weight."""
+ assert gate.scaled_reserve(20.0, 60.0, 60.0) == 20.0
+ assert gate.scaled_reserve(20.0, 30.0, 60.0) == 10.0
+ assert gate.scaled_reserve(10.0, 30.0, 60.0) == 5.0
+
+
+def test_an_unknown_plan_size_leaves_the_reserve_alone():
+ """Scaling by a total nobody could read would invent a number. The flat
+ value is the conservative answer and is what is kept."""
+ assert gate.scaled_reserve(20.0, 0.0, 60.0) == 20.0
+ assert gate.scaled_reserve(20.0, 30.0, 0.0) == 20.0
+
+
+# ------------------------------------------------------------- the in-flight
+
+
+def test_a_sweep_leaves_the_other_account_s_kernels_filed(tmp_path, monkeypatch):
+ """This token cannot delete that kernel. Trying turns a real leak into a
+ log line that reads exactly like a kernel already gone -- and DROPS the
+ record, which is the only thing that still knows the kernel exists."""
+ registry = tmp_path / "inflight.json"
+ registry.write_text(
+ json.dumps(
+ [
+ {"slug": "alice/unsloth-t4-ci-aaaa", "pid": 999999, "at": 0, "owner": "alice"},
+ {"slug": "bob/unsloth-t4-ci-bbbb", "pid": 999999, "at": 0, "owner": "bob"},
+ ]
+ ),
+ encoding = "utf-8",
+ )
+ monkeypatch.setattr(launch, "INFLIGHT", registry)
+ monkeypatch.setattr(launch, "_pid_alive", lambda pid: False)
+
+ attempted: list[str] = []
+
+ def _delete(slug, *_a, **_k):
+ attempted.append(slug)
+ return True
+
+ monkeypatch.setattr(launch, "delete_kernel", _delete, raising = False)
+ monkeypatch.setattr(
+ launch.subprocess,
+ "run",
+ lambda *a, **k: _delete(a[0][3]) and types_simple(),
+ raising = False,
+ )
+
+ launch.sweep_orphans("alice")
+
+ assert all(
+ "bob/" not in slug for slug in attempted
+ ), f"the sweep tried to delete another account's kernel: {attempted}"
+ left = {e["slug"] for e in json.loads(registry.read_text(encoding = "utf-8"))}
+ assert (
+ "bob/unsloth-t4-ci-bbbb" in left
+ ), "the other account's kernel was dropped from the registry, so nothing knows it exists"
+
+
+class types_simple: # noqa: N801 - a stand-in CompletedProcess
+ returncode = 0
+ stdout = ""
+ stderr = ""
+
+
+def _run_launcher(monkeypatch, tmp_path, username, *, user_arg):
+ """`launch.main()` with Kaggle stubbed, up to the point it decides to push."""
+ outdir = tmp_path / "out"
+ pushed: list = []
+
+ class _Api:
+ CONFIG_NAME_USER = "username"
+
+ def __init__(self):
+ self.config_values = {"username": username} if username else {}
+
+ monkeypatch.setattr(launch, "_api", lambda *a, **k: _Api())
+ monkeypatch.setattr(launch, "sweep_orphans", lambda *a, **k: [])
+ monkeypatch.setattr(
+ launch,
+ "push",
+ lambda *a, **k: pushed.append(a) or {"ok": False, "reason": "stub", "attempts": []},
+ )
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "launch.py",
+ "--notebook",
+ str(tmp_path / "k.ipynb"),
+ "--user",
+ user_arg,
+ "--outdir",
+ str(outdir),
+ ],
+ )
+ code = launch.main()
+ result = json.loads((outdir / "launch_result.json").read_text(encoding = "utf-8"))
+ return code, result, pushed
+
+
+def test_the_launcher_refuses_a_username_the_token_does_not_own(tmp_path, monkeypatch):
+ """The cross-check behind the matrix, and it must REFUSE rather than log.
+
+ If the selected account and the token ever disagree, every push fails for a
+ reason that reads like a bad notebook -- or succeeds under a name whose
+ kernels this job's cleanup then cannot delete, and the session bills on with
+ nobody watching. Nothing may be pushed in that state.
+ """
+ code, result, pushed = _run_launcher(monkeypatch, tmp_path, "alice", user_arg = "bob")
+ assert pushed == [], "a kernel was pushed under a name the token does not own"
+ assert result["verdict"] == "infra"
+ assert "alice" in result["reason"] and "bob" in result["reason"], result["reason"]
+
+
+def test_the_launcher_refuses_a_token_that_cannot_name_its_account(tmp_path, monkeypatch):
+ """No owner, no push. The owner is not optional: it is half the kernel id."""
+ code, result, pushed = _run_launcher(monkeypatch, tmp_path, None, user_arg = "bob")
+ assert pushed == []
+ assert "could not determine which Kaggle account" in result["reason"]
+
+
+def test_the_stand_down_still_writes_the_result_the_report_step_reads(tmp_path, monkeypatch):
+ """A bare `return 1` here would leave no launch_result.json, and the report
+ step reads that file: a configuration error would arrive looking exactly
+ like a runner that died mid-run."""
+ _, result, _ = _run_launcher(monkeypatch, tmp_path, "alice", user_arg = "bob")
+ assert result["verdict"] == "infra" and result["reason"]
+
+
+def test_a_filed_slug_records_the_account_that_owns_it(tmp_path, monkeypatch):
+ monkeypatch.setattr(launch, "INFLIGHT", tmp_path / "inflight.json")
+ launch._inflight_add("carol/unsloth-t4-ci-cccc")
+ entry = json.loads((tmp_path / "inflight.json").read_text(encoding = "utf-8"))[0]
+ assert entry["owner"] == "carol", entry
+
+
+# -------------------------------------------------------------- the leak set
+
+
+def test_neither_token_name_can_reach_the_kernel():
+ """The built notebook is what Kaggle receives. Both account env vars belong
+ in the forbidden list, not just the first one."""
+ source = (REPO_ROOT / "tests" / "kaggle" / "test_t4_smoke_harness.py").read_text(
+ encoding = "utf-8"
+ )
+ for name in gate.DEFAULT_ACCOUNT_ENVS:
+ assert (
+ f'"{name}"' in source
+ ), f"{name} is not in the credential-leak guard, so a kernel could carry it"