Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

test: skip the CPU bf16 autocast expert-stream case on Windows CI runners (illegal instruction) (#2167)

Alpamys Makazhan0ec5a1e2f401
diff --git a/tests/_windows_ci.py b/tests/_windows_ci.py
index 83909d65..539ce202 100644
--- a/tests/_windows_ci.py
+++ b/tests/_windows_ci.py
@@ -1,8 +1,10 @@
"""THE ``windows-latest`` illegal-instruction guard (#382). One definition.
-Imported by the test files whose real ``trainer.train()`` still crashes that
-fleet: today only the two NF4 tests in ``test_v07202.py``. It lives here rather
-than in that file because the guard has been needed in four files over time, and
+Imported by the test files whose native kernels still crash that fleet: the two
+NF4 ``trainer.train()`` tests in ``test_v07202.py``, the CPU bf16-autocast
+case of ``test_expert_stream.py`` and the bfloat16 cases of
+``test_stream_loss_chunk.py``. It lives here rather than in one of them
+because the guard has been needed in four files over time, and
this repo has been bitten three times by a predicate that was copied instead of
shared (#372, #392, #424). ``test_windows_ci_guard_is_not_
duplicated`` fails if a second copy appears.
@@ -33,7 +35,14 @@ def _windows_ci() -> bool:
instruction the bitsandbytes wheel emits, and those tests still use this skip.
For the others it was the first CPU bf16/fp16 matmul in oneDNN or MKL (#1314),
which the ``aten_half_matmuls`` fixture avoids, so they request the fixture
- instead. Occurrences so far span py3.10, py3.11 and py3.12, so it tracks
+ instead. There are two exceptions. The first is the ``cpu-bf16-autocast`` case of
+ ``test_expert_stream.py::test_streamed_experts_backward_preserves_forward_autocast``
+ (run 40003126970, the crash frame is the eager module's bf16 forward): that
+ single case is skipped here, nothing else in the file. The bfloat16 cases of
+ ``test_stream_loss_chunk.py`` are the second (run 40014485149, the crash
+ frame is a bf16 ``Linear.forward`` with no autocast): those parametrisations
+ are skipped here, the float32 ones are not. Occurrences so far span
+ py3.10, py3.11 and py3.12, so it tracks
the runner CPU rather than the interpreter; the control is that ``1715261``
is the crashing tree ``7c9a931`` plus one line of markdown and came back
green on the same pool.
diff --git a/tests/test_expert_stream.py b/tests/test_expert_stream.py
index dd78a54a..5cef8c1b 100644
--- a/tests/test_expert_stream.py
+++ b/tests/test_expert_stream.py
@@ -9,6 +9,8 @@ from __future__ import annotations
import pytest
+from tests._windows_ci import skip_on_windows_ci
+
CUDA = pytest.mark.gpu(reason="NF4 dequantisation needs bitsandbytes on CUDA")
@@ -124,7 +126,18 @@ def test_streamed_experts_backward_equals_eager_bit_exactly(tmp_path, build):
reader.close()
[email protected]("autocast", [False, True], ids=["fp32-control", "cpu-bf16-autocast"])
+# The bf16 case runs a CPU bf16 matmul under autocast, and part of GitHub's windows-latest
+# fleet dies on it with 0xc000001d (illegal instruction) in the eager reference module's
+# forward; the crash kills the xdist worker and the cell later times out (#382, #1314).
+# Only that case is skipped, and only on a Windows CI runner: the fp32 control, a developer's
+# Windows box, ubuntu and macos all still run it.
+ "autocast",
+ [
+ pytest.param(False, id="fp32-control"),
+ pytest.param(True, id="cpu-bf16-autocast", marks=skip_on_windows_ci),
+ ],
+)
def test_streamed_experts_backward_preserves_forward_autocast(tmp_path, autocast):
"""Recomputation must differentiate the forward precision after its AMP region exits."""
import torch
diff --git a/tests/test_issue382_windows_ci_guard_is_shared.py b/tests/test_issue382_windows_ci_guard_is_shared.py
index f10ff616..799ab2c8 100644
--- a/tests/test_issue382_windows_ci_guard_is_shared.py
+++ b/tests/test_issue382_windows_ci_guard_is_shared.py
@@ -8,12 +8,14 @@ original guard did not cover. Copying the predicate would have been the third
time this repo shipped a duplicated source of truth (#372, #392, #424), so it
moved to ``tests/_windows_ci.py`` and both call sites import it.
-These tests exist so a future copy fails loudly instead of drifting. One file
-carries the guard now: the two NF4 tests in ``test_v07202.py``, whose crash is
-the bitsandbytes wheel. The others traced to CPU bf16/fp16 matmuls (#1314) and
-request ``aten_half_matmuls`` instead (#1359). Both sets are named here, in
-:data:`GUARDED_TESTS` and :data:`FIXTURE_TESTS`, so a deleted decorator or a
-dropped fixture fails here too.
+These tests exist so a future copy fails loudly instead of drifting. Three files
+carry the guard now: the two NF4 tests in ``test_v07202.py``, whose crash is
+the bitsandbytes wheel, the ``cpu-bf16-autocast`` case (that case only) of
+``test_expert_stream.py``, and the bfloat16 cases (those only) of
+``test_stream_loss_chunk.py``. The others traced to CPU bf16/fp16 matmuls (#1314) and
+request ``aten_half_matmuls`` instead (#1359). The sets are named here, in
+:data:`GUARDED_TESTS`, :data:`GUARDED_PARAMS` and :data:`FIXTURE_TESTS`, so a
+deleted decorator or a dropped fixture fails here too.
"""
from __future__ import annotations
@@ -30,10 +32,12 @@ from tests._windows_ci import _windows_ci, skip_on_windows_ci
_TESTS_DIR = Path(__file__).resolve().parent
_HOME = _TESTS_DIR / "_windows_ci.py"
-#: The files that gate a real ``trainer.train()`` behind the shared guard. One
-#: now: #382 (v07202). v05311, test_rewind_hf and test_rewind_wiring dropped it
-#: in #1359, once #1314 traced their crash to CPU half-precision matmuls.
-KNOWN_CALL_SITES = ("test_v07202.py",)
+#: The files that gate a crashing native kernel behind the shared guard: #382
+#: (v07202, a real ``trainer.train()``), the CPU bf16-autocast case of
+#: test_expert_stream and the CPU bfloat16 cases of test_stream_loss_chunk. v05311,
+#: test_rewind_hf and test_rewind_wiring dropped it in #1359, once #1314 traced their
+#: crash to CPU half-precision matmuls.
+KNOWN_CALL_SITES = ("test_v07202.py", "test_expert_stream.py", "test_stream_loss_chunk.py")
class TestThereIsExactlyOneDefinition:
@@ -120,6 +124,24 @@ GUARDED_TESTS: dict[str, tuple[str, ...]] = {
),
}
+#: Tests that skip ONE parametrisation, not the whole test: file -> test -> the
+#: ``pytest.param`` ids that must carry the marker. Every other id of that test
+#: must carry none, and the test itself must not be decorated: the fp32 control
+#: is the half that proves the comparison is not vacuous, and it never reaches
+#: a half-precision kernel.
+GUARDED_PARAMS: dict[str, dict[str, tuple[str, ...]]] = {
+ "test_expert_stream.py": {
+ "test_streamed_experts_backward_preserves_forward_autocast": (
+ "cpu-bf16-autocast",
+ ),
+ },
+ "test_stream_loss_chunk.py": {
+ "test_they_equal_the_full_logits_step_within_the_tolerance": ("bf16",),
+ "test_one_chunk_is_the_full_step_exactly": ("bf16",),
+ "test_the_token_losses_give_the_rewind_logs_row_losses": ("bf16",),
+ },
+}
+
#: Every test that runs a real CPU bf16/fp16 step and requests
#: ``aten_half_matmuls`` instead of the skip (#1314, #1359). Dropping the fixture
#: from one of them sends its matmuls back to oneDNN / MKL, which is the path that
@@ -151,6 +173,26 @@ def _param_names(node: ast.FunctionDef) -> set:
return {arg.arg for arg in node.args.args}
+def _is_call_to(node: ast.AST, name: str) -> bool:
+ return isinstance(node, ast.Call) and getattr(node.func, "attr", "") == name
+
+
+def _param_marks(node: ast.FunctionDef) -> dict:
+ """``pytest.param(..., id=...)`` id -> its ``marks`` source (None if it has none)."""
+ found = {}
+ for decorator in node.decorator_list:
+ if not _is_call_to(decorator, "parametrize"):
+ continue
+ for call in ast.walk(decorator):
+ if _is_call_to(call, "param"):
+ keywords = {kw.arg: kw.value for kw in call.keywords}
+ marks = keywords.get("marks")
+ found[ast.literal_eval(keywords["id"])] = (
+ None if marks is None else ast.unparse(marks)
+ )
+ return found
+
+
def _find_test(filename: str, target: str) -> ast.FunctionDef:
tree = ast.parse((_TESTS_DIR / filename).read_text(encoding="utf-8"))
found = [
@@ -177,8 +219,153 @@ class TestTheCrashingTestsAreActuallyGuarded:
)
def test_every_guarded_file_is_a_known_call_site(self):
- """The two lists cannot drift apart."""
- assert set(GUARDED_TESTS) == set(KNOWN_CALL_SITES)
+ """The lists cannot drift apart."""
+ assert set(GUARDED_TESTS) | set(GUARDED_PARAMS) == set(KNOWN_CALL_SITES)
+
+
+def _marker_condition(platform: str, ci: str | None) -> bool:
+ """The condition ``skip_on_windows_ci`` has on a host with this platform and ``CI``.
+
+ The marker's condition is computed once, when ``tests/_windows_ci.py`` is
+ imported, so patching the platform afterwards changes nothing. Executing the
+ module's source in a throwaway namespace evaluates it again without touching
+ the real module (a ``reload`` would swap the marker object that every test file
+ already holds).
+ """
+ source = _HOME.read_text(encoding="utf-8")
+ namespace = {"__name__": "_windows_ci_probe"}
+ with pytest.MonkeyPatch.context() as patch:
+ patch.setattr(sys, "platform", platform)
+ if ci is None:
+ patch.delenv("CI", raising=False)
+ else:
+ patch.setenv("CI", ci)
+ exec(compile(source, str(_HOME), "exec"), namespace)
+ return namespace["skip_on_windows_ci"].args[0]
+
+
+class TestTheCpuBf16SkipsStayNarrow:
+ """The skips on the CPU bf16 cases must exclude one CI fleet, nothing more.
+
+ Run 40003126970 (windows 3.12, PR #1855): the eager reference module's bf16
+ forward under CPU autocast died with ``0xc000001d``, the xdist worker was
+ replaced and the cell timed out 45 minutes later. Run 40014485149 (job
+ 120622979972, this PR's own windows cell) died the same way in
+ ``test_stream_loss_chunk.py``, on the bf16 head's ``Linear.forward`` of
+ ``test_they_equal_the_full_logits_step_within_the_tolerance[16-shape0-dtype1]``.
+ A skip that quietly widened would remove the case from ubuntu and macos too and
+ look exactly like a pass, so both edges of the condition are pinned on the marker
+ itself, plus which parametrisations of each test carry it.
+ """
+
+ def test_it_skips_on_a_windows_runner(self):
+ assert _marker_condition("win32", "true") is True
+
+ def test_it_runs_on_a_local_windows_box(self):
+ """The maintainer's Windows CPU is known to run bf16 autocast."""
+ assert _marker_condition("win32", None) is False
+
+ def test_it_runs_on_linux_ci(self):
+ assert _marker_condition("linux", "true") is False
+
+ def test_it_runs_on_macos_ci(self):
+ assert _marker_condition("darwin", "true") is False
+
+ @pytest.mark.parametrize(
+ "filename, target, skipped",
+ [
+ (filename, target, ids)
+ for filename, targets in GUARDED_PARAMS.items()
+ for target, ids in targets.items()
+ ],
+ )
+ def test_only_the_named_cases_carry_the_marker(self, filename, target, skipped):
+ node = _find_test(filename, target)
+ assert "skip_on_windows_ci" not in _decorator_names(node), (
+ f"{filename}::{target} is decorated, which skips every case; put the marker "
+ f"on the pytest.param of {skipped} instead"
+ )
+ cases = _param_marks(node)
+ assert set(skipped) <= set(cases), (
+ f"{filename}::{target} has no pytest.param with id in {skipped}: {sorted(cases)}"
+ )
+ for case, marks in cases.items():
+ if case in skipped:
+ assert marks == "skip_on_windows_ci", (case, marks)
+ else:
+ assert marks is None, f"{filename}::{target}[{case}] must keep running: {marks}"
+
+
+def _bf16_sites(filename: str):
+ """Where ``torch.bfloat16`` appears in a test file, by test.
+
+ Returns ``(by_test, strays)``: ``by_test`` maps each test that lists the dtype in a
+ ``parametrize`` to one bool per occurrence, True when it is the first argument of a
+ ``pytest.param`` whose ``marks`` is exactly ``skip_on_windows_ci``; ``strays`` counts
+ the occurrences outside any ``parametrize`` decorator (a test body that builds a bf16
+ head by hand would run on every cell, marker or not).
+ """
+ tree = ast.parse((_TESTS_DIR / filename).read_text(encoding="utf-8"))
+ every = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Attribute) and node.attr == "bfloat16"
+ ]
+ by_test: dict[str, list[bool]] = {}
+ in_parametrize = set()
+ for func in ast.walk(tree):
+ if not isinstance(func, ast.FunctionDef):
+ continue
+ for decorator in func.decorator_list:
+ if not _is_call_to(decorator, "parametrize"):
+ continue
+ guarded = set()
+ for call in ast.walk(decorator):
+ if _is_call_to(call, "param") and call.args:
+ marks = {kw.arg: ast.unparse(kw.value) for kw in call.keywords}.get("marks")
+ if marks == "skip_on_windows_ci":
+ guarded.add(id(call.args[0]))
+ for node in ast.walk(decorator):
+ if isinstance(node, ast.Attribute) and node.attr == "bfloat16":
+ in_parametrize.add(id(node))
+ by_test.setdefault(func.name, []).append(id(node) in guarded)
+ strays = sum(1 for node in every if id(node) not in in_parametrize)
+ return by_test, strays
+
+
+class TestEveryBf16CaseOfTheLossChunkTestsIsSkippedOnWindowsCI:
+ """Run 40014485149 died on the first bf16 case, and every other one is the same matmul.
+
+ ``GUARDED_PARAMS`` pins the ids that carry the marker on the three tests that list
+ bfloat16 today. This reads the file the other way round, so that a FOURTH test with a
+ bfloat16 parameter, or a bf16 head built inside a test body, cannot arrive unguarded
+ and bring the Windows cell down again.
+ """
+
+ FILE = "test_stream_loss_chunk.py"
+
+ def test_each_bfloat16_parameter_carries_the_marker(self):
+ by_test, _ = _bf16_sites(self.FILE)
+ unguarded = {name: flags for name, flags in by_test.items() if not all(flags)}
+ assert not unguarded, (
+ f"{self.FILE}: a bfloat16 parameter without marks=skip_on_windows_ci runs a "
+ f"CPU bf16 matmul on every Windows CI runner (#382): {unguarded}"
+ )
+
+ def test_exactly_the_pinned_tests_list_bfloat16(self):
+ by_test, _ = _bf16_sites(self.FILE)
+ assert set(by_test) == set(GUARDED_PARAMS[self.FILE]), (
+ f"{self.FILE}: tests with a bfloat16 parameter {sorted(by_test)} differ from the "
+ f"pinned {sorted(GUARDED_PARAMS[self.FILE])}; add the new one to GUARDED_PARAMS"
+ )
+ assert all(len(flags) == 1 for flags in by_test.values()), by_test
+
+ def test_no_bfloat16_outside_a_parametrize(self):
+ _, strays = _bf16_sites(self.FILE)
+ assert strays == 0, (
+ f"{self.FILE}: {strays} torch.bfloat16 outside a parametrize runs unguarded; "
+ f"parametrize it with pytest.param(..., id=..., marks=skip_on_windows_ci)"
+ )
class TestTheHalfPrecisionTestsRequestTheFixture:
diff --git a/tests/test_stream_loss_chunk.py b/tests/test_stream_loss_chunk.py
index 91fea73c..6c012010 100644
--- a/tests/test_stream_loss_chunk.py
+++ b/tests/test_stream_loss_chunk.py
@@ -11,6 +11,8 @@ from __future__ import annotations
import pytest
+from tests._windows_ci import skip_on_windows_ci
+
torch = pytest.importorskip("torch")
pytest.importorskip("transformers")
@@ -18,6 +20,15 @@ pytest.importorskip("transformers")
# float32 (2 x 67 x 101, chunk 16) and 0 in bfloat16 over the same shapes.
_RTOL = 1e-5
+# The bfloat16 cases run a plain CPU bf16 matmul (``torch.nn.Linear.forward`` on the head,
+# in ``_full``), and part of GitHub's windows-latest fleet dies on it with 0xc000001d
+# (illegal instruction): the interpreter exits, the xdist worker is replaced and the cell
+# later times out (#382, #1314; run 40014485149, job 120622979972, the crash frame is the
+# head's forward inside _full). Only the bfloat16 cases carry the marker, and it skips them on
+# a Windows CI runner only: the float32 cases, a developer's Windows box, ubuntu and macos
+# run everything. Each parametrisation spells its two pytest.param out, with an id, so that
+# tests/test_issue382_windows_ci_guard_is_shared.py can read which ones carry the marker.
+
def _full(head, hidden, labels, **kwargs):
from transformers.loss.loss_utils import ForCausalLMLoss
@@ -72,7 +83,13 @@ def _close(got, want):
class TestTheLossAndItsGradient:
- @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
+ @pytest.mark.parametrize(
+ "dtype",
+ [
+ pytest.param(torch.float32, id="fp32"),
+ pytest.param(torch.bfloat16, id="bf16", marks=skip_on_windows_ci),
+ ],
+ )
@pytest.mark.parametrize(
("chunk", "shape"),
[
@@ -91,7 +108,13 @@ class TestTheLossAndItsGradient:
assert float(full_grad.abs().max()) > 0 # the premise: there is a gradient
assert _close(grad, full_grad), float((grad - full_grad).abs().max())
- @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
+ @pytest.mark.parametrize(
+ "dtype",
+ [
+ pytest.param(torch.float32, id="fp32"),
+ pytest.param(torch.bfloat16, id="bf16", marks=skip_on_windows_ci),
+ ],
+ )
def test_one_chunk_is_the_full_step_exactly(self, dtype):
"""With nothing to reorder, there is nothing to round differently."""
(full_loss, full_grad), (loss, grad) = _both(dtype, 67)
@@ -252,7 +275,13 @@ class TestTheMetrics:
assert int(step.correct) == int(((predictions == shifted) & mask).sum())
assert 0 < int(step.correct) < int(step.labelled) # the premise
- @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
+ @pytest.mark.parametrize(
+ "dtype",
+ [
+ pytest.param(torch.float32, id="fp32"),
+ pytest.param(torch.bfloat16, id="bf16", marks=skip_on_windows_ci),
+ ],
+ )
def test_the_token_losses_give_the_rewind_logs_row_losses(self, dtype):
"""``row_losses`` is what the rewind log computes from the logits: the
mean token cross-entropy of each row, in the logits' own dtype."""