Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kaggle T4: split the load phase into fetch and weight load (#9692)

Daniel Hane18a069c15cd
diff --git a/.github/scripts/kaggle_t4_ci/legs.py b/.github/scripts/kaggle_t4_ci/legs.py
index 3e9f6c42dec..f0cda2a2c86 100644
--- a/.github/scripts/kaggle_t4_ci/legs.py
+++ b/.github/scripts/kaggle_t4_ci/legs.py
@@ -188,7 +188,15 @@ class Leg:
# attributable to a version; the canary dataset; and the "did the optimizer
# apply anything" evidence, without which every payload here can pass on a run
# that trained nothing.
-COMMON_FILES = ("versions.py", "canary_dataset.jsonl", "training_evidence.py")
+COMMON_FILES = (
+ "versions.py",
+ "canary_dataset.jsonl",
+ "training_evidence.py",
+ # Splits the one `load` figure into fetch and weight load. Shipped to every
+ # leg rather than to the ones that use it today, because the number it
+ # produces is only comparable across legs if every leg can produce it.
+ "phase_timers.py",
+)
# The install prefix shared by every leg. unsloth_zoo first and WITH deps, then
# unsloth on top, then bitsandbytes, which neither pulls and the image does not
diff --git a/.github/scripts/kaggle_t4_ci/report.py b/.github/scripts/kaggle_t4_ci/report.py
index d694deee783..7f727eb2b97 100644
--- a/.github/scripts/kaggle_t4_ci/report.py
+++ b/.github/scripts/kaggle_t4_ci/report.py
@@ -130,7 +130,7 @@ def own_verdict(kernel_verdict: str, kernel_reason: str, reports: list, expect:
def render(report: dict) -> list[str]:
lines = [
- f"#### payload `{report.get('label', '?')}` " f"model `{report.get('model', '?')}`",
+ f"#### payload `{report.get('label', '?')}` model `{report.get('model', '?')}`",
"",
]
if report.get("probe"):
@@ -248,7 +248,7 @@ def render(report: dict) -> list[str]:
compiled = report.get("compile")
if compiled:
if not compiled.get("available"):
- lines.append(f"torch.compile: **unreadable** - " f"{compiled.get('error')}")
+ lines.append(f"torch.compile: **unreadable** - {compiled.get('error')}")
else:
lines.append(
f"torch.compile: {compiled.get('unique_graphs')} unique "
@@ -262,6 +262,29 @@ def render(report: dict) -> list[str]:
)
lines.append("")
+ # The whole point of the instrumentation: a reader answers "is the Hub
+ # download worth optimising" from the job summary, without downloading the
+ # evidence artifact. `fetch_seconds` is None when the timer never attached,
+ # and that is rendered as its own sentence rather than as a zero -- "no
+ # download happened" and "nothing was measured" are different findings.
+ phases = report.get("load_phases")
+ if phases:
+ if phases.get("fetch_seconds") is None:
+ lines.append(
+ f"Load {phases.get('total_seconds')}s, split unknown: the fetch "
+ f"timer never attached, so this run says nothing about it."
+ )
+ else:
+ rate = phases.get("fetch_mb_s")
+ lines.append(
+ f"Load {phases.get('total_seconds')}s = fetch "
+ f"{phases.get('fetch_seconds')}s "
+ f"({phases.get('fetch_mb')} MB"
+ + (f" at {rate} MB/s" if rate else "")
+ + f") + weight load {phases.get('weight_load_seconds')}s."
+ )
+ lines.append("")
+
memory = report.get("memory_peak") or report.get("memory_after_train")
if memory:
lines.append(
@@ -454,7 +477,7 @@ def main() -> int:
for kernel in result.get("kernels") or []:
if kernel.get("slug"):
lines.append(
- f"Kernel: `{kernel['slug']}` (private), terminal " f"state `{kernel.get('state')}`."
+ f"Kernel: `{kernel['slug']}` (private), terminal state `{kernel.get('state')}`."
)
else:
lines.append(
@@ -463,8 +486,7 @@ def main() -> int:
)
if not result.get("kernels") and result.get("slug"):
lines.append(
- f"Kernel: `{result['slug']}` (private), terminal state "
- f"`{result.get('kernel_state')}`."
+ f"Kernel: `{result['slug']}` (private), terminal state `{result.get('kernel_state')}`."
)
lines.append("")
@@ -489,8 +511,7 @@ def main() -> int:
"causes: the Kaggle account was at its 2-kernel concurrency cap, "
"the weekly GPU quota was exhausted, or the push was throttled.",
"",
- "Re-run with the `kaggle-t4-ci` label or a manual dispatch to "
- "force another attempt.",
+ "Re-run with the `kaggle-t4-ci` label or a manual dispatch to force another attempt.",
]
_summary("\n".join(lines))
diff --git a/.github/workflows/kaggle-t4-notebook-ci.yml b/.github/workflows/kaggle-t4-notebook-ci.yml
index dc2659e69e5..012ac41a0c3 100644
--- a/.github/workflows/kaggle-t4-notebook-ci.yml
+++ b/.github/workflows/kaggle-t4-notebook-ci.yml
@@ -704,7 +704,12 @@ jobs:
# success on the step, because two vision-dataset guards raised
# ModuleNotFoundError. test_harness_dependencies.py keeps this line and
# that import set in agreement.
- python -m pip install --quiet pytest transformers peft datasets pillow pyyaml
+ # huggingface_hub is named rather than left to arrive under
+ # transformers: test_phase_timers.py imports it directly to stand in
+ # for the download entry points the fetch timer wraps, and this line's
+ # whole history is transitive arrivals being luck rather than a
+ # dependency.
+ python -m pip install --quiet pytest transformers peft datasets pillow pyyaml huggingface_hub
python -c "import torch, sys; sys.exit(None if torch.version.cuda is None else 'torch %s is built for CUDA %s: the CPU-index install was skipped as already satisfied' % (torch.__version__, torch.version.cuda))"
python -m pytest tests/kaggle -q
diff --git a/tests/kaggle/t4_smoke/phase_timers.py b/tests/kaggle/t4_smoke/phase_timers.py
new file mode 100644
index 00000000000..93634bd20ef
--- /dev/null
+++ b/tests/kaggle/t4_smoke/phase_timers.py
@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Split the load phase into FETCH and WEIGHT LOAD, so one dispatch can say
+whether the Hub download is worth optimising at all.
+
+Every leg today reports one `load` number covering `from_pretrained`, which is
+a download, a disk read and a quantised materialisation in one figure. The
+prefetch lane, the second-wave reorder and the whole `D` argument in the plan
+rest on how that figure splits, and it has never been measured -- only bounded
+by subtraction, which is not the same thing.
+
+**Why this instruments the process rather than the cache.** The obvious
+approach is to diff the hub cache around the call. It cannot work here: the
+legs download into ONE shared cache CONCURRENTLY, so a whole-cache delta
+credits another leg's bytes to this one and reports a rate the Hub never
+delivered. Scoping the diff to one repo folder fixes that but reintroduces a
+different error, because `load_in_4bit=True` redirects through Unsloth's
+FLOAT_TO_INT_MAPPER and the repo that downloads is not the repo that was asked
+for. Timing the download calls INSIDE this interpreter is immune to both: each
+leg is its own process, so anything measured here was fetched by this leg.
+
+**The failure mode this is built to avoid.** An instrument that silently
+reports zero is worse than no instrument, because "no download happened" and
+"the timer never attached" read identically in a report and only one of them is
+a finding. So `seconds` is None until something is genuinely patched, and
+`patched` names what was wrapped. A reader can tell the two apart.
+
+Nothing here may fail the run. A payload that dies while collecting a
+diagnostic reports nothing at all, which is the one outcome worse than a
+missing number.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import time
+
+# The download entry points transformers and unsloth actually reach. Every one
+# is patched where present, and the transformers aliases are NOT redundant:
+# `transformers.utils.hub` does `from huggingface_hub import ...` at import
+# time, so it holds its own reference and rebinding the public name leaves it
+# untouched. `cached_files` calls its module-level `snapshot_download` for a
+# multi-file (sharded) checkpoint, which is the dominant download in every leg
+# here, so omitting that alias moves the largest fetch into weight_load_seconds
+# while `patched` stays non-empty and the record still looks valid.
+_TARGETS = (
+ ("huggingface_hub", "hf_hub_download"),
+ ("huggingface_hub", "snapshot_download"),
+ ("transformers.utils.hub", "hf_hub_download"),
+ ("transformers.utils.hub", "snapshot_download"),
+)
+
+
+class FetchTimer:
+ """Accumulate wall time and bytes spent inside Hub download calls.
+
+ Re-entrant by design: `snapshot_download` calls `hf_hub_download` per file,
+ so a naive sum would count the inner calls twice and report more download
+ seconds than the phase itself took. A depth counter means only the
+ OUTERMOST call contributes to `seconds`.
+ """
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._depth = 0
+ self._seconds = 0.0
+ self.calls = 0
+ self.bytes = 0
+ self.patched: list = []
+ self._originals: list = []
+
+ # -------------------------------------------------------------- patching
+
+ def _wrap(self, fn):
+ def wrapped(*args, **kwargs):
+ with self._lock:
+ outermost = self._depth == 0
+ self._depth += 1
+ self.calls += 1
+ started = time.time()
+ try:
+ result = fn(*args, **kwargs)
+ finally:
+ with self._lock:
+ self._depth -= 1
+ if outermost:
+ self._seconds += time.time() - started
+ # The returned path is the file or directory that now exists, so
+ # its size is what this call put on disk. A warm cache returns the
+ # same path having transferred nothing, which reads as its own size
+ # rather than as zero -- accepted, and stated, because the leg runs
+ # against a cache the prefetch may already have warmed and the
+ # interesting number there is `seconds`, not `bytes`.
+ try:
+ self.bytes += _path_bytes(result)
+ except Exception: # noqa: BLE001
+ pass
+ return result
+
+ return wrapped
+
+ def install(self) -> "FetchTimer":
+ import importlib
+ for module_name, attr in _TARGETS:
+ try:
+ module = importlib.import_module(module_name)
+ except Exception: # noqa: BLE001
+ continue
+ original = getattr(module, attr, None)
+ if original is None or not callable(original):
+ continue
+ try:
+ setattr(module, attr, self._wrap(original))
+ except Exception: # noqa: BLE001
+ continue
+ self._originals.append((module, attr, original))
+ self.patched.append(f"{module_name}.{attr}")
+ return self
+
+ def uninstall(self) -> None:
+ for module, attr, original in self._originals:
+ try:
+ setattr(module, attr, original)
+ except Exception: # noqa: BLE001
+ pass
+ self._originals = []
+
+ # --------------------------------------------------------------- reading
+
+ @property
+ def seconds(self):
+ """None when nothing was patched, so a dead timer cannot read as 0.0."""
+ if not self.patched:
+ return None
+ return round(self._seconds, 1)
+
+ def record(self, total_seconds: float) -> dict:
+ """The split, plus enough context to distrust it if it deserves that."""
+ fetch = self.seconds
+ out = {
+ "patched": list(self.patched),
+ "calls": self.calls,
+ "fetch_seconds": fetch,
+ "fetch_mb": round(self.bytes / 1024**2, 1) if self.patched else None,
+ "total_seconds": round(total_seconds, 1),
+ }
+ if fetch is None:
+ out["weight_load_seconds"] = None
+ out["note"] = (
+ "the fetch timer never attached, so this run says nothing about "
+ "the split; do not read the absence as 'no download happened'"
+ )
+ return out
+ # Clamped at zero: the two clocks are the same clock, but rounding and
+ # a download finishing inside the final microseconds of the phase can
+ # still produce a negative by a tenth, and a negative duration in a
+ # report is read as a bug in the report rather than as rounding.
+ out["weight_load_seconds"] = round(max(total_seconds - self._seconds, 0.0), 1)
+ if self.bytes and self._seconds > 0:
+ out["fetch_mb_s"] = round(self.bytes / 1024**2 / self._seconds, 1)
+ return out
+
+ def __enter__(self) -> "FetchTimer":
+ return self.install()
+
+ def __exit__(self, *_exc) -> None:
+ self.uninstall()
+
+
+def _path_bytes(path) -> int:
+ if not path:
+ return 0
+ path = str(path)
+ if os.path.isfile(path):
+ return os.path.getsize(path)
+ if not os.path.isdir(path):
+ return 0
+ total = 0
+ for dirpath, _dirnames, filenames in os.walk(path):
+ for name in filenames:
+ try:
+ total += os.stat(os.path.join(dirpath, name)).st_size
+ except OSError:
+ pass
+ return total
diff --git a/tests/kaggle/t4_smoke/run_t4_smoke.py b/tests/kaggle/t4_smoke/run_t4_smoke.py
index d8ffd77b8c9..ef5a9d2332a 100644
--- a/tests/kaggle/t4_smoke/run_t4_smoke.py
+++ b/tests/kaggle/t4_smoke/run_t4_smoke.py
@@ -265,6 +265,8 @@ def train_once(args, run_index: int) -> dict:
rows = load_canary_rows(Path(args.dataset))
+ from phase_timers import FetchTimer
+
t0 = time.time()
# float16 unconditionally: T4 is sm_75 and has no bf16. Pinned rather than
# left to the loader so the local reproduction and the Kaggle run take the
@@ -297,14 +299,25 @@ def train_once(args, run_index: int) -> dict:
# gets switched off before the day it is right. The binding coverage --
# the whole reason this leg exists -- does not depend on it.
load_kwargs["device_map"] = {"": 0}
- model, tokenizer = FastLanguageModel.from_pretrained(
- model_name = args.model,
- max_seq_length = args.max_seq_length,
- load_in_4bit = True,
- dtype = torch.float16,
- **load_kwargs,
- )
+ #
+ # The timer splits this one number into fetch and weight load. `load` on
+ # its own cannot say whether the prefetch lane is buying anything.
+ with FetchTimer() as fetch_timer:
+ model, tokenizer = FastLanguageModel.from_pretrained(
+ model_name = args.model,
+ max_seq_length = args.max_seq_length,
+ load_in_4bit = True,
+ dtype = torch.float16,
+ **load_kwargs,
+ )
load_seconds = time.time() - t0
+ load_phases = fetch_timer.record(load_seconds)
+ _log(
+ "load phases: "
+ + f"fetch={load_phases['fetch_seconds']}s "
+ + f"weight_load={load_phases['weight_load_seconds']}s "
+ + f"patched={len(load_phases['patched'])}"
+ )
# Which repository, and which commit of it. `load_in_4bit=True` redirects
# through Unsloth's FLOAT_TO_INT_MAPPER, so the config name is not the name
# asked for, and the loader explicitly DROPS `revision=` once a remap
@@ -615,6 +628,10 @@ def train_once(args, run_index: int) -> dict:
"save": round(save_seconds, 1),
"infer": round(infer_seconds, 1),
},
+ # `load` split into the download and everything after it. Kept beside
+ # the totals rather than replacing them, so the existing series stays
+ # comparable across runs that predate this.
+ "load_phases": load_phases,
"peak_reserved_gb": round(peak_gb, 2),
}
diff --git a/tests/kaggle/test_phase_timers.py b/tests/kaggle/test_phase_timers.py
new file mode 100644
index 00000000000..a8d781d7885
--- /dev/null
+++ b/tests/kaggle/test_phase_timers.py
@@ -0,0 +1,266 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The load phase split, and the ways a timer like this reports a lie.
+
+Three of these guards exist because the obvious implementation gets them wrong
+and the wrong answer looks exactly like a real result in the report:
+
+* a timer that never attached reporting **0.0 seconds**, which reads as "no
+ download happened" rather than "nothing was measured";
+* `snapshot_download` calling `hf_hub_download` per file, so a naive sum counts
+ the same seconds twice and can report more download time than the phase it
+ sat inside;
+* a raising download leaving the timer installed, so every later call in the
+ process is still wrapped.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+import types
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+PAYLOAD = ROOT / "tests" / "kaggle" / "t4_smoke"
+sys.path.insert(0, str(PAYLOAD))
+
+from phase_timers import FetchTimer # noqa: E402
+
+
+def hub(monkeypatch):
+ """A stand-in `huggingface_hub` with the two entry points the timer wraps."""
+ module = types.ModuleType("huggingface_hub")
+ module.hf_hub_download = lambda *a, **k: ""
+ module.snapshot_download = lambda *a, **k: ""
+ monkeypatch.setitem(sys.modules, "huggingface_hub", module)
+ return module
+
+
+def test_an_unattached_timer_reports_none_not_zero(monkeypatch):
+ """The finding this whole file exists for. With nothing patched, `seconds`
+ must be None: a report showing 0.0 is indistinguishable from a warm cache,
+ and one of those is a measurement while the other is a broken instrument."""
+ monkeypatch.setitem(sys.modules, "huggingface_hub", types.ModuleType("huggingface_hub"))
+ monkeypatch.delitem(sys.modules, "transformers.utils.hub", raising = False)
+ timer = FetchTimer().install()
+ try:
+ assert timer.patched == []
+ assert timer.seconds is None
+ record = timer.record(12.0)
+ assert record["fetch_seconds"] is None
+ assert record["weight_load_seconds"] is None
+ assert "do not read the absence" in record["note"]
+ finally:
+ timer.uninstall()
+
+
+def test_a_real_download_is_timed_and_sized(hub, tmp_path):
+ blob = tmp_path / "model.safetensors"
+ blob.write_bytes(b"x" * 4096)
+
+ def slow_download(*_a, **_k):
+ time.sleep(0.05)
+ return str(blob)
+
+ hub.hf_hub_download = slow_download
+ with FetchTimer() as timer:
+ import huggingface_hub
+ huggingface_hub.hf_hub_download(repo_id = "org/model")
+ record = timer.record(1.0)
+ assert record["calls"] == 1
+ assert record["fetch_seconds"] >= 0.0
+ assert timer.bytes == 4096
+ assert record["weight_load_seconds"] == round(1.0 - timer._seconds, 1)
+
+
+def test_nested_calls_are_not_counted_twice(hub, tmp_path):
+ """`snapshot_download` calls `hf_hub_download` per file. Without the depth
+ counter the inner calls add their own seconds again, and the phase can
+ report more download time than it lasted."""
+ blob = tmp_path / "f.bin"
+ blob.write_bytes(b"y" * 10)
+
+ def inner(*_a, **_k):
+ time.sleep(0.03)
+ return str(blob)
+
+ def outer(*_a, **_k):
+ import huggingface_hub
+ for _ in range(3):
+ huggingface_hub.hf_hub_download()
+ return str(tmp_path)
+
+ hub.hf_hub_download = inner
+ hub.snapshot_download = outer
+
+ with FetchTimer() as timer:
+ import huggingface_hub
+
+ started = time.time()
+ huggingface_hub.snapshot_download(repo_id = "org/model")
+ elapsed = time.time() - started
+
+ assert timer.calls == 4, "every call is counted"
+ # Against the outer call's OWN elapsed time, not a fixed ceiling: the three
+ # inner sleeps run inside that call, so double counting lands near 2x it
+ # while a correct sum lands at 1x. A host that deschedules the runner
+ # stretches both sides equally, where a constant bound would go red on
+ # scheduling and report it as double counting.
+ assert (
+ timer._seconds <= elapsed * 1.5
+ ), f"nested seconds counted twice: {timer._seconds} against {elapsed} elapsed"
+
+
+def test_a_raising_download_still_restores_the_module(hub):
+ def boom(*_a, **_k):
+ raise RuntimeError("hub is down")
+
+ hub.hf_hub_download = boom
+ original = hub.hf_hub_download
+ timer = FetchTimer().install()
+ import huggingface_hub
+
+ with pytest.raises(RuntimeError):
+ huggingface_hub.hf_hub_download()
+ timer.uninstall()
+ assert huggingface_hub.hf_hub_download is original, "the wrapper outlived the timer"
+ # The failed attempt is still time spent trying, so it counts.
+ assert timer.calls == 1
+
+
+def test_the_split_never_reports_a_negative_weight_load(hub, tmp_path):
+ """The two clocks are the same clock, but rounding can still put the fetch a
+ tenth past the phase, and a negative duration reads as a broken report."""
+ blob = tmp_path / "f.bin"
+ blob.write_bytes(b"z")
+
+ def slow(*_a, **_k):
+ time.sleep(0.2)
+ return str(blob)
+
+ hub.hf_hub_download = slow
+ with FetchTimer() as timer:
+ import huggingface_hub
+ huggingface_hub.hf_hub_download()
+ record = timer.record(0.05)
+ assert record["weight_load_seconds"] == 0.0
+
+
+def test_no_alias_of_a_hub_download_is_left_unwrapped(monkeypatch):
+ """`transformers.utils.hub` does `from huggingface_hub import ...` at import
+ time, so it holds its OWN reference and rebinding the public name leaves it
+ untouched. `cached_files` calls that alias for a multi-file (sharded)
+ checkpoint, which is the biggest download any leg does, so missing it moves
+ the dominant fetch into `weight_load_seconds` while `patched` stays
+ non-empty and the record still looks valid.
+
+ Derived, not listed: the aliases are DISCOVERED by comparing each module's
+ attributes against the originals before patching, so a module that starts
+ holding one of these names is covered without editing this test, and
+ dropping a target fails here rather than on hardware.
+ """
+ real_hub = types.ModuleType("huggingface_hub")
+ real_hub.hf_hub_download = lambda *a, **k: ""
+ real_hub.snapshot_download = lambda *a, **k: ""
+ # Exactly what `from huggingface_hub import snapshot_download` produces.
+ alias = types.ModuleType("transformers.utils.hub")
+ alias.hf_hub_download = real_hub.hf_hub_download
+ alias.snapshot_download = real_hub.snapshot_download
+ monkeypatch.setitem(sys.modules, "huggingface_hub", real_hub)
+ monkeypatch.setitem(sys.modules, "transformers.utils.hub", alias)
+
+ originals = {
+ (name, attr): getattr(module, attr)
+ for name, module in (("huggingface_hub", real_hub), ("transformers.utils.hub", alias))
+ for attr in ("hf_hub_download", "snapshot_download")
+ if callable(getattr(module, attr, None))
+ }
+ assert len(originals) == 4, "the fixture no longer models the alias"
+
+ timer = FetchTimer().install()
+ try:
+ unwrapped = sorted(
+ f"{name}.{attr}"
+ for (name, attr), original in originals.items()
+ if getattr(sys.modules[name], attr) is original
+ )
+ finally:
+ timer.uninstall()
+ assert unwrapped == [], f"these download entry points are never timed: {unwrapped}"
+
+ for (name, attr), original in originals.items():
+ assert getattr(sys.modules[name], attr) is original, f"{name}.{attr} outlived the timer"
+
+
+def _render(report: dict) -> str:
+ """The real renderer, loaded by path: `.github/scripts` is not a package."""
+ import importlib.util
+
+ path = ROOT / ".github" / "scripts" / "kaggle_t4_ci" / "report.py"
+ spec = importlib.util.spec_from_file_location("_t4_report_for_test", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return "\n".join(module.render(report))
+
+
+def test_the_job_summary_shows_the_split_it_was_added_to_answer():
+ """A number that only reaches `launch_result.json` answers nobody: reading it
+ means downloading the evidence artifact, which is not where anyone looks."""
+ rendered = _render(
+ {
+ "label": "control",
+ "load_phases": {
+ "patched": ["huggingface_hub.hf_hub_download"],
+ "fetch_seconds": 61.7,
+ "fetch_mb": 12550.0,
+ "fetch_mb_s": 203.4,
+ "weight_load_seconds": 40.9,
+ "total_seconds": 102.6,
+ },
+ }
+ )
+ assert "61.7" in rendered, "the fetch half is missing from the summary"
+ assert "40.9" in rendered, "the weight-load half is missing from the summary"
+ assert "203.4" in rendered, "the achieved rate is missing from the summary"
+
+
+def test_an_unattached_timer_is_not_rendered_as_a_zero_second_fetch():
+ """The failure the instrument was built around, at the reporting layer: a
+ timer that never attached must not read as 'the download took no time'."""
+ rendered = _render(
+ {
+ "label": "control",
+ "load_phases": {
+ "patched": [],
+ "fetch_seconds": None,
+ "fetch_mb": None,
+ "weight_load_seconds": None,
+ "total_seconds": 102.6,
+ },
+ }
+ )
+ assert "never attached" in rendered
+ assert "= fetch" not in rendered, "a split was rendered from a timer that measured nothing"
+
+
+def test_every_leg_ships_the_module():
+ """A timer the payload cannot import measures nothing. Asserted against the
+ registry rather than a hardcoded list, so a new leg cannot miss it."""
+ sys.path.insert(0, str(ROOT / ".github" / "scripts"))
+ from kaggle_t4_ci.legs import LEGS
+
+ missing = sorted(n for n, leg in LEGS.items() if "phase_timers.py" not in leg.files)
+ assert missing == [], f"legs that cannot import phase_timers: {missing}"
+
+
+def test_the_payload_actually_calls_the_timer():
+ """A module shipped and never used is coverage that does nothing."""
+ src = (PAYLOAD / "run_t4_smoke.py").read_text(encoding = "utf-8")
+ assert "from phase_timers import FetchTimer" in src
+ assert "with FetchTimer() as fetch_timer:" in src
+ assert '"load_phases": load_phases' in src
diff --git a/tests/kaggle/test_t4_smoke_harness.py b/tests/kaggle/test_t4_smoke_harness.py
index d2c24a20d30..5dd87aba271 100644
--- a/tests/kaggle/test_t4_smoke_harness.py
+++ b/tests/kaggle/test_t4_smoke_harness.py
@@ -2160,10 +2160,11 @@ def test_the_files_the_payload_carries_are_byte_identical_to_the_repo(tmp_path):
payload = _payload_notebooks(_build(tmp_path))["t4_control.ipynb"]
blob = re.search(r"^FILES = (\{.*?\})$", _cell(payload, 0), re.M | re.S).group(1)
files = json.loads(blob)
- assert set(files) == {
+ expected = {
"versions.py",
"canary_dataset.jsonl",
"training_evidence.py",
+ "phase_timers.py",
"run_t4_smoke.py",
"determinism.py",
"gguf_export.py",