Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
test: skip the CPU bf16 autocast expert-stream case on Windows CI runners (illegal instruction) (#2167)
Alpamys Makazhan0ec5a1e2f401
diff --git a/tests/_windows_ci.py b/tests/_windows_ci.pyindex 83909d65..539ce202 100644--- a/tests/_windows_ci.py+++ b/tests/_windows_ci.py@@ -1,8 +1,10 @@"""THE ``windows-latest`` illegal-instruction guard (#382). One definition.-Imported by the test files whose real ``trainer.train()`` still crashes that-fleet: today only the two NF4 tests in ``test_v07202.py``. It lives here rather-than in that file because the guard has been needed in four files over time, and+Imported by the test files whose native kernels still crash that fleet: the two+NF4 ``trainer.train()`` tests in ``test_v07202.py``, the CPU bf16-autocast+case of ``test_expert_stream.py`` and the bfloat16 cases of+``test_stream_loss_chunk.py``. It lives here rather than in one of them+because the guard has been needed in four files over time, andthis repo has been bitten three times by a predicate that was copied instead ofshared (#372, #392, #424). ``test_windows_ci_guard_is_not_duplicated`` fails if a second copy appears.@@ -33,7 +35,14 @@ def _windows_ci() -> bool:instruction the bitsandbytes wheel emits, and those tests still use this skip.For the others it was the first CPU bf16/fp16 matmul in oneDNN or MKL (#1314),which the ``aten_half_matmuls`` fixture avoids, so they request the fixture- instead. Occurrences so far span py3.10, py3.11 and py3.12, so it tracks+ instead. There are two exceptions. The first is the ``cpu-bf16-autocast`` case of+ ``test_expert_stream.py::test_streamed_experts_backward_preserves_forward_autocast``+ (run 40003126970, the crash frame is the eager module's bf16 forward): that+ single case is skipped here, nothing else in the file. The bfloat16 cases of+ ``test_stream_loss_chunk.py`` are the second (run 40014485149, the crash+ frame is a bf16 ``Linear.forward`` with no autocast): those parametrisations+ are skipped here, the float32 ones are not. Occurrences so far span+ py3.10, py3.11 and py3.12, so it tracksthe runner CPU rather than the interpreter; the control is that ``1715261``is the crashing tree ``7c9a931`` plus one line of markdown and came backgreen on the same pool.diff --git a/tests/test_expert_stream.py b/tests/test_expert_stream.pyindex dd78a54a..5cef8c1b 100644--- a/tests/test_expert_stream.py+++ b/tests/test_expert_stream.py@@ -9,6 +9,8 @@ from __future__ import annotationsimport pytest+from tests._windows_ci import skip_on_windows_ci+CUDA = pytest.mark.gpu(reason="NF4 dequantisation needs bitsandbytes on CUDA")@@ -124,7 +126,18 @@ def test_streamed_experts_backward_equals_eager_bit_exactly(tmp_path, build):reader.close()[email protected]("autocast", [False, True], ids=["fp32-control", "cpu-bf16-autocast"])+# The bf16 case runs a CPU bf16 matmul under autocast, and part of GitHub's windows-latest+# fleet dies on it with 0xc000001d (illegal instruction) in the eager reference module's+# forward; the crash kills the xdist worker and the cell later times out (#382, #1314).+# Only that case is skipped, and only on a Windows CI runner: the fp32 control, a developer's+# Windows box, ubuntu and macos all still run it.+ "autocast",+ [+ pytest.param(False, id="fp32-control"),+ pytest.param(True, id="cpu-bf16-autocast", marks=skip_on_windows_ci),+ ],+)def test_streamed_experts_backward_preserves_forward_autocast(tmp_path, autocast):"""Recomputation must differentiate the forward precision after its AMP region exits."""import torchdiff --git a/tests/test_issue382_windows_ci_guard_is_shared.py b/tests/test_issue382_windows_ci_guard_is_shared.pyindex f10ff616..799ab2c8 100644--- a/tests/test_issue382_windows_ci_guard_is_shared.py+++ b/tests/test_issue382_windows_ci_guard_is_shared.py@@ -8,12 +8,14 @@ original guard did not cover. Copying the predicate would have been the thirdtime this repo shipped a duplicated source of truth (#372, #392, #424), so itmoved to ``tests/_windows_ci.py`` and both call sites import it.-These tests exist so a future copy fails loudly instead of drifting. One file-carries the guard now: the two NF4 tests in ``test_v07202.py``, whose crash is-the bitsandbytes wheel. The others traced to CPU bf16/fp16 matmuls (#1314) and-request ``aten_half_matmuls`` instead (#1359). Both sets are named here, in-:data:`GUARDED_TESTS` and :data:`FIXTURE_TESTS`, so a deleted decorator or a-dropped fixture fails here too.+These tests exist so a future copy fails loudly instead of drifting. Three files+carry the guard now: the two NF4 tests in ``test_v07202.py``, whose crash is+the bitsandbytes wheel, the ``cpu-bf16-autocast`` case (that case only) of+``test_expert_stream.py``, and the bfloat16 cases (those only) of+``test_stream_loss_chunk.py``. The others traced to CPU bf16/fp16 matmuls (#1314) and+request ``aten_half_matmuls`` instead (#1359). The sets are named here, in+:data:`GUARDED_TESTS`, :data:`GUARDED_PARAMS` and :data:`FIXTURE_TESTS`, so a+deleted decorator or a dropped fixture fails here too."""from __future__ import annotations@@ -30,10 +32,12 @@ from tests._windows_ci import _windows_ci, skip_on_windows_ci_TESTS_DIR = Path(__file__).resolve().parent_HOME = _TESTS_DIR / "_windows_ci.py"-#: The files that gate a real ``trainer.train()`` behind the shared guard. One-#: now: #382 (v07202). v05311, test_rewind_hf and test_rewind_wiring dropped it-#: in #1359, once #1314 traced their crash to CPU half-precision matmuls.-KNOWN_CALL_SITES = ("test_v07202.py",)+#: The files that gate a crashing native kernel behind the shared guard: #382+#: (v07202, a real ``trainer.train()``), the CPU bf16-autocast case of+#: test_expert_stream and the CPU bfloat16 cases of test_stream_loss_chunk. v05311,+#: test_rewind_hf and test_rewind_wiring dropped it in #1359, once #1314 traced their+#: crash to CPU half-precision matmuls.+KNOWN_CALL_SITES = ("test_v07202.py", "test_expert_stream.py", "test_stream_loss_chunk.py")class TestThereIsExactlyOneDefinition:@@ -120,6 +124,24 @@ GUARDED_TESTS: dict[str, tuple[str, ...]] = {),}+#: Tests that skip ONE parametrisation, not the whole test: file -> test -> the+#: ``pytest.param`` ids that must carry the marker. Every other id of that test+#: must carry none, and the test itself must not be decorated: the fp32 control+#: is the half that proves the comparison is not vacuous, and it never reaches+#: a half-precision kernel.+GUARDED_PARAMS: dict[str, dict[str, tuple[str, ...]]] = {+ "test_expert_stream.py": {+ "test_streamed_experts_backward_preserves_forward_autocast": (+ "cpu-bf16-autocast",+ ),+ },+ "test_stream_loss_chunk.py": {+ "test_they_equal_the_full_logits_step_within_the_tolerance": ("bf16",),+ "test_one_chunk_is_the_full_step_exactly": ("bf16",),+ "test_the_token_losses_give_the_rewind_logs_row_losses": ("bf16",),+ },+}+#: Every test that runs a real CPU bf16/fp16 step and requests#: ``aten_half_matmuls`` instead of the skip (#1314, #1359). Dropping the fixture#: from one of them sends its matmuls back to oneDNN / MKL, which is the path that@@ -151,6 +173,26 @@ def _param_names(node: ast.FunctionDef) -> set:return {arg.arg for arg in node.args.args}+def _is_call_to(node: ast.AST, name: str) -> bool:+ return isinstance(node, ast.Call) and getattr(node.func, "attr", "") == name+++def _param_marks(node: ast.FunctionDef) -> dict:+ """``pytest.param(..., id=...)`` id -> its ``marks`` source (None if it has none)."""+ found = {}+ for decorator in node.decorator_list:+ if not _is_call_to(decorator, "parametrize"):+ continue+ for call in ast.walk(decorator):+ if _is_call_to(call, "param"):+ keywords = {kw.arg: kw.value for kw in call.keywords}+ marks = keywords.get("marks")+ found[ast.literal_eval(keywords["id"])] = (+ None if marks is None else ast.unparse(marks)+ )+ return found++def _find_test(filename: str, target: str) -> ast.FunctionDef:tree = ast.parse((_TESTS_DIR / filename).read_text(encoding="utf-8"))found = [@@ -177,8 +219,153 @@ class TestTheCrashingTestsAreActuallyGuarded:)def test_every_guarded_file_is_a_known_call_site(self):- """The two lists cannot drift apart."""- assert set(GUARDED_TESTS) == set(KNOWN_CALL_SITES)+ """The lists cannot drift apart."""+ assert set(GUARDED_TESTS) | set(GUARDED_PARAMS) == set(KNOWN_CALL_SITES)+++def _marker_condition(platform: str, ci: str | None) -> bool:+ """The condition ``skip_on_windows_ci`` has on a host with this platform and ``CI``.++ The marker's condition is computed once, when ``tests/_windows_ci.py`` is+ imported, so patching the platform afterwards changes nothing. Executing the+ module's source in a throwaway namespace evaluates it again without touching+ the real module (a ``reload`` would swap the marker object that every test file+ already holds).+ """+ source = _HOME.read_text(encoding="utf-8")+ namespace = {"__name__": "_windows_ci_probe"}+ with pytest.MonkeyPatch.context() as patch:+ patch.setattr(sys, "platform", platform)+ if ci is None:+ patch.delenv("CI", raising=False)+ else:+ patch.setenv("CI", ci)+ exec(compile(source, str(_HOME), "exec"), namespace)+ return namespace["skip_on_windows_ci"].args[0]+++class TestTheCpuBf16SkipsStayNarrow:+ """The skips on the CPU bf16 cases must exclude one CI fleet, nothing more.++ Run 40003126970 (windows 3.12, PR #1855): the eager reference module's bf16+ forward under CPU autocast died with ``0xc000001d``, the xdist worker was+ replaced and the cell timed out 45 minutes later. Run 40014485149 (job+ 120622979972, this PR's own windows cell) died the same way in+ ``test_stream_loss_chunk.py``, on the bf16 head's ``Linear.forward`` of+ ``test_they_equal_the_full_logits_step_within_the_tolerance[16-shape0-dtype1]``.+ A skip that quietly widened would remove the case from ubuntu and macos too and+ look exactly like a pass, so both edges of the condition are pinned on the marker+ itself, plus which parametrisations of each test carry it.+ """++ def test_it_skips_on_a_windows_runner(self):+ assert _marker_condition("win32", "true") is True++ def test_it_runs_on_a_local_windows_box(self):+ """The maintainer's Windows CPU is known to run bf16 autocast."""+ assert _marker_condition("win32", None) is False++ def test_it_runs_on_linux_ci(self):+ assert _marker_condition("linux", "true") is False++ def test_it_runs_on_macos_ci(self):+ assert _marker_condition("darwin", "true") is False++ @pytest.mark.parametrize(+ "filename, target, skipped",+ [+ (filename, target, ids)+ for filename, targets in GUARDED_PARAMS.items()+ for target, ids in targets.items()+ ],+ )+ def test_only_the_named_cases_carry_the_marker(self, filename, target, skipped):+ node = _find_test(filename, target)+ assert "skip_on_windows_ci" not in _decorator_names(node), (+ f"{filename}::{target} is decorated, which skips every case; put the marker "+ f"on the pytest.param of {skipped} instead"+ )+ cases = _param_marks(node)+ assert set(skipped) <= set(cases), (+ f"{filename}::{target} has no pytest.param with id in {skipped}: {sorted(cases)}"+ )+ for case, marks in cases.items():+ if case in skipped:+ assert marks == "skip_on_windows_ci", (case, marks)+ else:+ assert marks is None, f"{filename}::{target}[{case}] must keep running: {marks}"+++def _bf16_sites(filename: str):+ """Where ``torch.bfloat16`` appears in a test file, by test.++ Returns ``(by_test, strays)``: ``by_test`` maps each test that lists the dtype in a+ ``parametrize`` to one bool per occurrence, True when it is the first argument of a+ ``pytest.param`` whose ``marks`` is exactly ``skip_on_windows_ci``; ``strays`` counts+ the occurrences outside any ``parametrize`` decorator (a test body that builds a bf16+ head by hand would run on every cell, marker or not).+ """+ tree = ast.parse((_TESTS_DIR / filename).read_text(encoding="utf-8"))+ every = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Attribute) and node.attr == "bfloat16"+ ]+ by_test: dict[str, list[bool]] = {}+ in_parametrize = set()+ for func in ast.walk(tree):+ if not isinstance(func, ast.FunctionDef):+ continue+ for decorator in func.decorator_list:+ if not _is_call_to(decorator, "parametrize"):+ continue+ guarded = set()+ for call in ast.walk(decorator):+ if _is_call_to(call, "param") and call.args:+ marks = {kw.arg: ast.unparse(kw.value) for kw in call.keywords}.get("marks")+ if marks == "skip_on_windows_ci":+ guarded.add(id(call.args[0]))+ for node in ast.walk(decorator):+ if isinstance(node, ast.Attribute) and node.attr == "bfloat16":+ in_parametrize.add(id(node))+ by_test.setdefault(func.name, []).append(id(node) in guarded)+ strays = sum(1 for node in every if id(node) not in in_parametrize)+ return by_test, strays+++class TestEveryBf16CaseOfTheLossChunkTestsIsSkippedOnWindowsCI:+ """Run 40014485149 died on the first bf16 case, and every other one is the same matmul.++ ``GUARDED_PARAMS`` pins the ids that carry the marker on the three tests that list+ bfloat16 today. This reads the file the other way round, so that a FOURTH test with a+ bfloat16 parameter, or a bf16 head built inside a test body, cannot arrive unguarded+ and bring the Windows cell down again.+ """++ FILE = "test_stream_loss_chunk.py"++ def test_each_bfloat16_parameter_carries_the_marker(self):+ by_test, _ = _bf16_sites(self.FILE)+ unguarded = {name: flags for name, flags in by_test.items() if not all(flags)}+ assert not unguarded, (+ f"{self.FILE}: a bfloat16 parameter without marks=skip_on_windows_ci runs a "+ f"CPU bf16 matmul on every Windows CI runner (#382): {unguarded}"+ )++ def test_exactly_the_pinned_tests_list_bfloat16(self):+ by_test, _ = _bf16_sites(self.FILE)+ assert set(by_test) == set(GUARDED_PARAMS[self.FILE]), (+ f"{self.FILE}: tests with a bfloat16 parameter {sorted(by_test)} differ from the "+ f"pinned {sorted(GUARDED_PARAMS[self.FILE])}; add the new one to GUARDED_PARAMS"+ )+ assert all(len(flags) == 1 for flags in by_test.values()), by_test++ def test_no_bfloat16_outside_a_parametrize(self):+ _, strays = _bf16_sites(self.FILE)+ assert strays == 0, (+ f"{self.FILE}: {strays} torch.bfloat16 outside a parametrize runs unguarded; "+ f"parametrize it with pytest.param(..., id=..., marks=skip_on_windows_ci)"+ )class TestTheHalfPrecisionTestsRequestTheFixture:diff --git a/tests/test_stream_loss_chunk.py b/tests/test_stream_loss_chunk.pyindex 91fea73c..6c012010 100644--- a/tests/test_stream_loss_chunk.py+++ b/tests/test_stream_loss_chunk.py@@ -11,6 +11,8 @@ from __future__ import annotationsimport pytest+from tests._windows_ci import skip_on_windows_ci+torch = pytest.importorskip("torch")pytest.importorskip("transformers")@@ -18,6 +20,15 @@ pytest.importorskip("transformers")# float32 (2 x 67 x 101, chunk 16) and 0 in bfloat16 over the same shapes._RTOL = 1e-5+# The bfloat16 cases run a plain CPU bf16 matmul (``torch.nn.Linear.forward`` on the head,+# in ``_full``), and part of GitHub's windows-latest fleet dies on it with 0xc000001d+# (illegal instruction): the interpreter exits, the xdist worker is replaced and the cell+# later times out (#382, #1314; run 40014485149, job 120622979972, the crash frame is the+# head's forward inside _full). Only the bfloat16 cases carry the marker, and it skips them on+# a Windows CI runner only: the float32 cases, a developer's Windows box, ubuntu and macos+# run everything. Each parametrisation spells its two pytest.param out, with an id, so that+# tests/test_issue382_windows_ci_guard_is_shared.py can read which ones carry the marker.+def _full(head, hidden, labels, **kwargs):from transformers.loss.loss_utils import ForCausalLMLoss@@ -72,7 +83,13 @@ def _close(got, want):class TestTheLossAndItsGradient:- @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])+ @pytest.mark.parametrize(+ "dtype",+ [+ pytest.param(torch.float32, id="fp32"),+ pytest.param(torch.bfloat16, id="bf16", marks=skip_on_windows_ci),+ ],+ )@pytest.mark.parametrize(("chunk", "shape"),[@@ -91,7 +108,13 @@ class TestTheLossAndItsGradient:assert float(full_grad.abs().max()) > 0 # the premise: there is a gradientassert _close(grad, full_grad), float((grad - full_grad).abs().max())- @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])+ @pytest.mark.parametrize(+ "dtype",+ [+ pytest.param(torch.float32, id="fp32"),+ pytest.param(torch.bfloat16, id="bf16", marks=skip_on_windows_ci),+ ],+ )def test_one_chunk_is_the_full_step_exactly(self, dtype):"""With nothing to reorder, there is nothing to round differently."""(full_loss, full_grad), (loss, grad) = _both(dtype, 67)@@ -252,7 +275,13 @@ class TestTheMetrics:assert int(step.correct) == int(((predictions == shifted) & mask).sum())assert 0 < int(step.correct) < int(step.labelled) # the premise- @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])+ @pytest.mark.parametrize(+ "dtype",+ [+ pytest.param(torch.float32, id="fp32"),+ pytest.param(torch.bfloat16, id="bf16", marks=skip_on_windows_ci),+ ],+ )def test_the_token_losses_give_the_rewind_logs_row_losses(self, dtype):"""``row_losses`` is what the rewind log computes from the logits: themean token cross-entropy of each row, in the logits' own dtype."""