Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kaggle T4: split the load phase into fetch and weight load (#9692)
Daniel Hane18a069c15cd
diff --git a/.github/scripts/kaggle_t4_ci/legs.py b/.github/scripts/kaggle_t4_ci/legs.pyindex 3e9f6c42dec..f0cda2a2c86 100644--- a/.github/scripts/kaggle_t4_ci/legs.py+++ b/.github/scripts/kaggle_t4_ci/legs.py@@ -188,7 +188,15 @@ class Leg:# attributable to a version; the canary dataset; and the "did the optimizer# apply anything" evidence, without which every payload here can pass on a run# that trained nothing.-COMMON_FILES = ("versions.py", "canary_dataset.jsonl", "training_evidence.py")+COMMON_FILES = (+ "versions.py",+ "canary_dataset.jsonl",+ "training_evidence.py",+ # Splits the one `load` figure into fetch and weight load. Shipped to every+ # leg rather than to the ones that use it today, because the number it+ # produces is only comparable across legs if every leg can produce it.+ "phase_timers.py",+)# The install prefix shared by every leg. unsloth_zoo first and WITH deps, then# unsloth on top, then bitsandbytes, which neither pulls and the image does notdiff --git a/.github/scripts/kaggle_t4_ci/report.py b/.github/scripts/kaggle_t4_ci/report.pyindex d694deee783..7f727eb2b97 100644--- a/.github/scripts/kaggle_t4_ci/report.py+++ b/.github/scripts/kaggle_t4_ci/report.py@@ -130,7 +130,7 @@ def own_verdict(kernel_verdict: str, kernel_reason: str, reports: list, expect:def render(report: dict) -> list[str]:lines = [- f"#### payload `{report.get('label', '?')}` " f"model `{report.get('model', '?')}`",+ f"#### payload `{report.get('label', '?')}` model `{report.get('model', '?')}`","",]if report.get("probe"):@@ -248,7 +248,7 @@ def render(report: dict) -> list[str]:compiled = report.get("compile")if compiled:if not compiled.get("available"):- lines.append(f"torch.compile: **unreadable** - " f"{compiled.get('error')}")+ lines.append(f"torch.compile: **unreadable** - {compiled.get('error')}")else:lines.append(f"torch.compile: {compiled.get('unique_graphs')} unique "@@ -262,6 +262,29 @@ def render(report: dict) -> list[str]:)lines.append("")+ # The whole point of the instrumentation: a reader answers "is the Hub+ # download worth optimising" from the job summary, without downloading the+ # evidence artifact. `fetch_seconds` is None when the timer never attached,+ # and that is rendered as its own sentence rather than as a zero -- "no+ # download happened" and "nothing was measured" are different findings.+ phases = report.get("load_phases")+ if phases:+ if phases.get("fetch_seconds") is None:+ lines.append(+ f"Load {phases.get('total_seconds')}s, split unknown: the fetch "+ f"timer never attached, so this run says nothing about it."+ )+ else:+ rate = phases.get("fetch_mb_s")+ lines.append(+ f"Load {phases.get('total_seconds')}s = fetch "+ f"{phases.get('fetch_seconds')}s "+ f"({phases.get('fetch_mb')} MB"+ + (f" at {rate} MB/s" if rate else "")+ + f") + weight load {phases.get('weight_load_seconds')}s."+ )+ lines.append("")+memory = report.get("memory_peak") or report.get("memory_after_train")if memory:lines.append(@@ -454,7 +477,7 @@ def main() -> int:for kernel in result.get("kernels") or []:if kernel.get("slug"):lines.append(- f"Kernel: `{kernel['slug']}` (private), terminal " f"state `{kernel.get('state')}`."+ f"Kernel: `{kernel['slug']}` (private), terminal state `{kernel.get('state')}`.")else:lines.append(@@ -463,8 +486,7 @@ def main() -> int:)if not result.get("kernels") and result.get("slug"):lines.append(- f"Kernel: `{result['slug']}` (private), terminal state "- f"`{result.get('kernel_state')}`."+ f"Kernel: `{result['slug']}` (private), terminal state `{result.get('kernel_state')}`.")lines.append("")@@ -489,8 +511,7 @@ def main() -> int:"causes: the Kaggle account was at its 2-kernel concurrency cap, ""the weekly GPU quota was exhausted, or the push was throttled.","",- "Re-run with the `kaggle-t4-ci` label or a manual dispatch to "- "force another attempt.",+ "Re-run with the `kaggle-t4-ci` label or a manual dispatch to force another attempt.",]_summary("\n".join(lines))diff --git a/.github/workflows/kaggle-t4-notebook-ci.yml b/.github/workflows/kaggle-t4-notebook-ci.ymlindex dc2659e69e5..012ac41a0c3 100644--- a/.github/workflows/kaggle-t4-notebook-ci.yml+++ b/.github/workflows/kaggle-t4-notebook-ci.yml@@ -704,7 +704,12 @@ jobs:# success on the step, because two vision-dataset guards raised# ModuleNotFoundError. test_harness_dependencies.py keeps this line and# that import set in agreement.- python -m pip install --quiet pytest transformers peft datasets pillow pyyaml+ # huggingface_hub is named rather than left to arrive under+ # transformers: test_phase_timers.py imports it directly to stand in+ # for the download entry points the fetch timer wraps, and this line's+ # whole history is transitive arrivals being luck rather than a+ # dependency.+ python -m pip install --quiet pytest transformers peft datasets pillow pyyaml huggingface_hubpython -c "import torch, sys; sys.exit(None if torch.version.cuda is None else 'torch %s is built for CUDA %s: the CPU-index install was skipped as already satisfied' % (torch.__version__, torch.version.cuda))"python -m pytest tests/kaggle -qdiff --git a/tests/kaggle/t4_smoke/phase_timers.py b/tests/kaggle/t4_smoke/phase_timers.pynew file mode 100644index 00000000000..93634bd20ef--- /dev/null+++ b/tests/kaggle/t4_smoke/phase_timers.py@@ -0,0 +1,187 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Split the load phase into FETCH and WEIGHT LOAD, so one dispatch can say+whether the Hub download is worth optimising at all.++Every leg today reports one `load` number covering `from_pretrained`, which is+a download, a disk read and a quantised materialisation in one figure. The+prefetch lane, the second-wave reorder and the whole `D` argument in the plan+rest on how that figure splits, and it has never been measured -- only bounded+by subtraction, which is not the same thing.++**Why this instruments the process rather than the cache.** The obvious+approach is to diff the hub cache around the call. It cannot work here: the+legs download into ONE shared cache CONCURRENTLY, so a whole-cache delta+credits another leg's bytes to this one and reports a rate the Hub never+delivered. Scoping the diff to one repo folder fixes that but reintroduces a+different error, because `load_in_4bit=True` redirects through Unsloth's+FLOAT_TO_INT_MAPPER and the repo that downloads is not the repo that was asked+for. Timing the download calls INSIDE this interpreter is immune to both: each+leg is its own process, so anything measured here was fetched by this leg.++**The failure mode this is built to avoid.** An instrument that silently+reports zero is worse than no instrument, because "no download happened" and+"the timer never attached" read identically in a report and only one of them is+a finding. So `seconds` is None until something is genuinely patched, and+`patched` names what was wrapped. A reader can tell the two apart.++Nothing here may fail the run. A payload that dies while collecting a+diagnostic reports nothing at all, which is the one outcome worse than a+missing number.+"""++from __future__ import annotations++import os+import threading+import time++# The download entry points transformers and unsloth actually reach. Every one+# is patched where present, and the transformers aliases are NOT redundant:+# `transformers.utils.hub` does `from huggingface_hub import ...` at import+# time, so it holds its own reference and rebinding the public name leaves it+# untouched. `cached_files` calls its module-level `snapshot_download` for a+# multi-file (sharded) checkpoint, which is the dominant download in every leg+# here, so omitting that alias moves the largest fetch into weight_load_seconds+# while `patched` stays non-empty and the record still looks valid.+_TARGETS = (+ ("huggingface_hub", "hf_hub_download"),+ ("huggingface_hub", "snapshot_download"),+ ("transformers.utils.hub", "hf_hub_download"),+ ("transformers.utils.hub", "snapshot_download"),+)+++class FetchTimer:+ """Accumulate wall time and bytes spent inside Hub download calls.++ Re-entrant by design: `snapshot_download` calls `hf_hub_download` per file,+ so a naive sum would count the inner calls twice and report more download+ seconds than the phase itself took. A depth counter means only the+ OUTERMOST call contributes to `seconds`.+ """++ def __init__(self) -> None:+ self._lock = threading.Lock()+ self._depth = 0+ self._seconds = 0.0+ self.calls = 0+ self.bytes = 0+ self.patched: list = []+ self._originals: list = []++ # -------------------------------------------------------------- patching++ def _wrap(self, fn):+ def wrapped(*args, **kwargs):+ with self._lock:+ outermost = self._depth == 0+ self._depth += 1+ self.calls += 1+ started = time.time()+ try:+ result = fn(*args, **kwargs)+ finally:+ with self._lock:+ self._depth -= 1+ if outermost:+ self._seconds += time.time() - started+ # The returned path is the file or directory that now exists, so+ # its size is what this call put on disk. A warm cache returns the+ # same path having transferred nothing, which reads as its own size+ # rather than as zero -- accepted, and stated, because the leg runs+ # against a cache the prefetch may already have warmed and the+ # interesting number there is `seconds`, not `bytes`.+ try:+ self.bytes += _path_bytes(result)+ except Exception: # noqa: BLE001+ pass+ return result++ return wrapped++ def install(self) -> "FetchTimer":+ import importlib+ for module_name, attr in _TARGETS:+ try:+ module = importlib.import_module(module_name)+ except Exception: # noqa: BLE001+ continue+ original = getattr(module, attr, None)+ if original is None or not callable(original):+ continue+ try:+ setattr(module, attr, self._wrap(original))+ except Exception: # noqa: BLE001+ continue+ self._originals.append((module, attr, original))+ self.patched.append(f"{module_name}.{attr}")+ return self++ def uninstall(self) -> None:+ for module, attr, original in self._originals:+ try:+ setattr(module, attr, original)+ except Exception: # noqa: BLE001+ pass+ self._originals = []++ # --------------------------------------------------------------- reading++ @property+ def seconds(self):+ """None when nothing was patched, so a dead timer cannot read as 0.0."""+ if not self.patched:+ return None+ return round(self._seconds, 1)++ def record(self, total_seconds: float) -> dict:+ """The split, plus enough context to distrust it if it deserves that."""+ fetch = self.seconds+ out = {+ "patched": list(self.patched),+ "calls": self.calls,+ "fetch_seconds": fetch,+ "fetch_mb": round(self.bytes / 1024**2, 1) if self.patched else None,+ "total_seconds": round(total_seconds, 1),+ }+ if fetch is None:+ out["weight_load_seconds"] = None+ out["note"] = (+ "the fetch timer never attached, so this run says nothing about "+ "the split; do not read the absence as 'no download happened'"+ )+ return out+ # Clamped at zero: the two clocks are the same clock, but rounding and+ # a download finishing inside the final microseconds of the phase can+ # still produce a negative by a tenth, and a negative duration in a+ # report is read as a bug in the report rather than as rounding.+ out["weight_load_seconds"] = round(max(total_seconds - self._seconds, 0.0), 1)+ if self.bytes and self._seconds > 0:+ out["fetch_mb_s"] = round(self.bytes / 1024**2 / self._seconds, 1)+ return out++ def __enter__(self) -> "FetchTimer":+ return self.install()++ def __exit__(self, *_exc) -> None:+ self.uninstall()+++def _path_bytes(path) -> int:+ if not path:+ return 0+ path = str(path)+ if os.path.isfile(path):+ return os.path.getsize(path)+ if not os.path.isdir(path):+ return 0+ total = 0+ for dirpath, _dirnames, filenames in os.walk(path):+ for name in filenames:+ try:+ total += os.stat(os.path.join(dirpath, name)).st_size+ except OSError:+ pass+ return totaldiff --git a/tests/kaggle/t4_smoke/run_t4_smoke.py b/tests/kaggle/t4_smoke/run_t4_smoke.pyindex d8ffd77b8c9..ef5a9d2332a 100644--- a/tests/kaggle/t4_smoke/run_t4_smoke.py+++ b/tests/kaggle/t4_smoke/run_t4_smoke.py@@ -265,6 +265,8 @@ def train_once(args, run_index: int) -> dict:rows = load_canary_rows(Path(args.dataset))+ from phase_timers import FetchTimer+t0 = time.time()# float16 unconditionally: T4 is sm_75 and has no bf16. Pinned rather than# left to the loader so the local reproduction and the Kaggle run take the@@ -297,14 +299,25 @@ def train_once(args, run_index: int) -> dict:# gets switched off before the day it is right. The binding coverage --# the whole reason this leg exists -- does not depend on it.load_kwargs["device_map"] = {"": 0}- model, tokenizer = FastLanguageModel.from_pretrained(- model_name = args.model,- max_seq_length = args.max_seq_length,- load_in_4bit = True,- dtype = torch.float16,- **load_kwargs,- )+ #+ # The timer splits this one number into fetch and weight load. `load` on+ # its own cannot say whether the prefetch lane is buying anything.+ with FetchTimer() as fetch_timer:+ model, tokenizer = FastLanguageModel.from_pretrained(+ model_name = args.model,+ max_seq_length = args.max_seq_length,+ load_in_4bit = True,+ dtype = torch.float16,+ **load_kwargs,+ )load_seconds = time.time() - t0+ load_phases = fetch_timer.record(load_seconds)+ _log(+ "load phases: "+ + f"fetch={load_phases['fetch_seconds']}s "+ + f"weight_load={load_phases['weight_load_seconds']}s "+ + f"patched={len(load_phases['patched'])}"+ )# Which repository, and which commit of it. `load_in_4bit=True` redirects# through Unsloth's FLOAT_TO_INT_MAPPER, so the config name is not the name# asked for, and the loader explicitly DROPS `revision=` once a remap@@ -615,6 +628,10 @@ def train_once(args, run_index: int) -> dict:"save": round(save_seconds, 1),"infer": round(infer_seconds, 1),},+ # `load` split into the download and everything after it. Kept beside+ # the totals rather than replacing them, so the existing series stays+ # comparable across runs that predate this.+ "load_phases": load_phases,"peak_reserved_gb": round(peak_gb, 2),}diff --git a/tests/kaggle/test_phase_timers.py b/tests/kaggle/test_phase_timers.pynew file mode 100644index 00000000000..a8d781d7885--- /dev/null+++ b/tests/kaggle/test_phase_timers.py@@ -0,0 +1,266 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The load phase split, and the ways a timer like this reports a lie.++Three of these guards exist because the obvious implementation gets them wrong+and the wrong answer looks exactly like a real result in the report:++* a timer that never attached reporting **0.0 seconds**, which reads as "no+ download happened" rather than "nothing was measured";+* `snapshot_download` calling `hf_hub_download` per file, so a naive sum counts+ the same seconds twice and can report more download time than the phase it+ sat inside;+* a raising download leaving the timer installed, so every later call in the+ process is still wrapped.+"""++from __future__ import annotations++import sys+import time+import types+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[2]+PAYLOAD = ROOT / "tests" / "kaggle" / "t4_smoke"+sys.path.insert(0, str(PAYLOAD))++from phase_timers import FetchTimer # noqa: E402+++def hub(monkeypatch):+ """A stand-in `huggingface_hub` with the two entry points the timer wraps."""+ module = types.ModuleType("huggingface_hub")+ module.hf_hub_download = lambda *a, **k: ""+ module.snapshot_download = lambda *a, **k: ""+ monkeypatch.setitem(sys.modules, "huggingface_hub", module)+ return module+++def test_an_unattached_timer_reports_none_not_zero(monkeypatch):+ """The finding this whole file exists for. With nothing patched, `seconds`+ must be None: a report showing 0.0 is indistinguishable from a warm cache,+ and one of those is a measurement while the other is a broken instrument."""+ monkeypatch.setitem(sys.modules, "huggingface_hub", types.ModuleType("huggingface_hub"))+ monkeypatch.delitem(sys.modules, "transformers.utils.hub", raising = False)+ timer = FetchTimer().install()+ try:+ assert timer.patched == []+ assert timer.seconds is None+ record = timer.record(12.0)+ assert record["fetch_seconds"] is None+ assert record["weight_load_seconds"] is None+ assert "do not read the absence" in record["note"]+ finally:+ timer.uninstall()+++def test_a_real_download_is_timed_and_sized(hub, tmp_path):+ blob = tmp_path / "model.safetensors"+ blob.write_bytes(b"x" * 4096)++ def slow_download(*_a, **_k):+ time.sleep(0.05)+ return str(blob)++ hub.hf_hub_download = slow_download+ with FetchTimer() as timer:+ import huggingface_hub+ huggingface_hub.hf_hub_download(repo_id = "org/model")+ record = timer.record(1.0)+ assert record["calls"] == 1+ assert record["fetch_seconds"] >= 0.0+ assert timer.bytes == 4096+ assert record["weight_load_seconds"] == round(1.0 - timer._seconds, 1)+++def test_nested_calls_are_not_counted_twice(hub, tmp_path):+ """`snapshot_download` calls `hf_hub_download` per file. Without the depth+ counter the inner calls add their own seconds again, and the phase can+ report more download time than it lasted."""+ blob = tmp_path / "f.bin"+ blob.write_bytes(b"y" * 10)++ def inner(*_a, **_k):+ time.sleep(0.03)+ return str(blob)++ def outer(*_a, **_k):+ import huggingface_hub+ for _ in range(3):+ huggingface_hub.hf_hub_download()+ return str(tmp_path)++ hub.hf_hub_download = inner+ hub.snapshot_download = outer++ with FetchTimer() as timer:+ import huggingface_hub++ started = time.time()+ huggingface_hub.snapshot_download(repo_id = "org/model")+ elapsed = time.time() - started++ assert timer.calls == 4, "every call is counted"+ # Against the outer call's OWN elapsed time, not a fixed ceiling: the three+ # inner sleeps run inside that call, so double counting lands near 2x it+ # while a correct sum lands at 1x. A host that deschedules the runner+ # stretches both sides equally, where a constant bound would go red on+ # scheduling and report it as double counting.+ assert (+ timer._seconds <= elapsed * 1.5+ ), f"nested seconds counted twice: {timer._seconds} against {elapsed} elapsed"+++def test_a_raising_download_still_restores_the_module(hub):+ def boom(*_a, **_k):+ raise RuntimeError("hub is down")++ hub.hf_hub_download = boom+ original = hub.hf_hub_download+ timer = FetchTimer().install()+ import huggingface_hub++ with pytest.raises(RuntimeError):+ huggingface_hub.hf_hub_download()+ timer.uninstall()+ assert huggingface_hub.hf_hub_download is original, "the wrapper outlived the timer"+ # The failed attempt is still time spent trying, so it counts.+ assert timer.calls == 1+++def test_the_split_never_reports_a_negative_weight_load(hub, tmp_path):+ """The two clocks are the same clock, but rounding can still put the fetch a+ tenth past the phase, and a negative duration reads as a broken report."""+ blob = tmp_path / "f.bin"+ blob.write_bytes(b"z")++ def slow(*_a, **_k):+ time.sleep(0.2)+ return str(blob)++ hub.hf_hub_download = slow+ with FetchTimer() as timer:+ import huggingface_hub+ huggingface_hub.hf_hub_download()+ record = timer.record(0.05)+ assert record["weight_load_seconds"] == 0.0+++def test_no_alias_of_a_hub_download_is_left_unwrapped(monkeypatch):+ """`transformers.utils.hub` does `from huggingface_hub import ...` at import+ time, so it holds its OWN reference and rebinding the public name leaves it+ untouched. `cached_files` calls that alias for a multi-file (sharded)+ checkpoint, which is the biggest download any leg does, so missing it moves+ the dominant fetch into `weight_load_seconds` while `patched` stays+ non-empty and the record still looks valid.++ Derived, not listed: the aliases are DISCOVERED by comparing each module's+ attributes against the originals before patching, so a module that starts+ holding one of these names is covered without editing this test, and+ dropping a target fails here rather than on hardware.+ """+ real_hub = types.ModuleType("huggingface_hub")+ real_hub.hf_hub_download = lambda *a, **k: ""+ real_hub.snapshot_download = lambda *a, **k: ""+ # Exactly what `from huggingface_hub import snapshot_download` produces.+ alias = types.ModuleType("transformers.utils.hub")+ alias.hf_hub_download = real_hub.hf_hub_download+ alias.snapshot_download = real_hub.snapshot_download+ monkeypatch.setitem(sys.modules, "huggingface_hub", real_hub)+ monkeypatch.setitem(sys.modules, "transformers.utils.hub", alias)++ originals = {+ (name, attr): getattr(module, attr)+ for name, module in (("huggingface_hub", real_hub), ("transformers.utils.hub", alias))+ for attr in ("hf_hub_download", "snapshot_download")+ if callable(getattr(module, attr, None))+ }+ assert len(originals) == 4, "the fixture no longer models the alias"++ timer = FetchTimer().install()+ try:+ unwrapped = sorted(+ f"{name}.{attr}"+ for (name, attr), original in originals.items()+ if getattr(sys.modules[name], attr) is original+ )+ finally:+ timer.uninstall()+ assert unwrapped == [], f"these download entry points are never timed: {unwrapped}"++ for (name, attr), original in originals.items():+ assert getattr(sys.modules[name], attr) is original, f"{name}.{attr} outlived the timer"+++def _render(report: dict) -> str:+ """The real renderer, loaded by path: `.github/scripts` is not a package."""+ import importlib.util++ path = ROOT / ".github" / "scripts" / "kaggle_t4_ci" / "report.py"+ spec = importlib.util.spec_from_file_location("_t4_report_for_test", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return "\n".join(module.render(report))+++def test_the_job_summary_shows_the_split_it_was_added_to_answer():+ """A number that only reaches `launch_result.json` answers nobody: reading it+ means downloading the evidence artifact, which is not where anyone looks."""+ rendered = _render(+ {+ "label": "control",+ "load_phases": {+ "patched": ["huggingface_hub.hf_hub_download"],+ "fetch_seconds": 61.7,+ "fetch_mb": 12550.0,+ "fetch_mb_s": 203.4,+ "weight_load_seconds": 40.9,+ "total_seconds": 102.6,+ },+ }+ )+ assert "61.7" in rendered, "the fetch half is missing from the summary"+ assert "40.9" in rendered, "the weight-load half is missing from the summary"+ assert "203.4" in rendered, "the achieved rate is missing from the summary"+++def test_an_unattached_timer_is_not_rendered_as_a_zero_second_fetch():+ """The failure the instrument was built around, at the reporting layer: a+ timer that never attached must not read as 'the download took no time'."""+ rendered = _render(+ {+ "label": "control",+ "load_phases": {+ "patched": [],+ "fetch_seconds": None,+ "fetch_mb": None,+ "weight_load_seconds": None,+ "total_seconds": 102.6,+ },+ }+ )+ assert "never attached" in rendered+ assert "= fetch" not in rendered, "a split was rendered from a timer that measured nothing"+++def test_every_leg_ships_the_module():+ """A timer the payload cannot import measures nothing. Asserted against the+ registry rather than a hardcoded list, so a new leg cannot miss it."""+ sys.path.insert(0, str(ROOT / ".github" / "scripts"))+ from kaggle_t4_ci.legs import LEGS++ missing = sorted(n for n, leg in LEGS.items() if "phase_timers.py" not in leg.files)+ assert missing == [], f"legs that cannot import phase_timers: {missing}"+++def test_the_payload_actually_calls_the_timer():+ """A module shipped and never used is coverage that does nothing."""+ src = (PAYLOAD / "run_t4_smoke.py").read_text(encoding = "utf-8")+ assert "from phase_timers import FetchTimer" in src+ assert "with FetchTimer() as fetch_timer:" in src+ assert '"load_phases": load_phases' in srcdiff --git a/tests/kaggle/test_t4_smoke_harness.py b/tests/kaggle/test_t4_smoke_harness.pyindex d2c24a20d30..5dd87aba271 100644--- a/tests/kaggle/test_t4_smoke_harness.py+++ b/tests/kaggle/test_t4_smoke_harness.py@@ -2160,10 +2160,11 @@ def test_the_files_the_payload_carries_are_byte_identical_to_the_repo(tmp_path):payload = _payload_notebooks(_build(tmp_path))["t4_control.ipynb"]blob = re.search(r"^FILES = (\{.*?\})$", _cell(payload, 0), re.M | re.S).group(1)files = json.loads(blob)- assert set(files) == {+ expected = {"versions.py","canary_dataset.jsonl","training_evidence.py",+ "phase_timers.py","run_t4_smoke.py","determinism.py","gguf_export.py",