Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ci): run the real bf16 forward in the child, not a stand-in

thread3d3717a00201e7
diff --git a/docs/reference/agent.md b/docs/reference/agent.md
index 90337a38..655f2e15 100644
--- a/docs/reference/agent.md
+++ b/docs/reference/agent.md
@@ -68,8 +68,12 @@ with torch.no_grad(), torch.autocast(device_type='cpu', dtype=torch.bfloat16):
print('bf16 CPU autocast runs here')"
```
-A non-zero exit with no traceback is the fault, not a failed assertion. `tests/test_runtime_fixes.py`
-runs exactly this probe before it asserts anything about the autocast path.
+A non-zero exit with no traceback is the fault, not a failed assertion. That probe only settles
+whether the instruction set can execute *a* bf16 kernel: oneDNN chooses per op and per shape, so it
+can pass on a host where Laya's own forward faults. To rule that out, put the agent you will serve
+with in a child process the same way. `tests/test_runtime_fixes.py` qualifies the CPU like that — it
+runs the model's own bf16 forward in a child and asserts on the numbers it prints back, so a
+faulting host costs three skipped checks rather than the suite.
## In-process dynamic quantization
diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py
index 31f85254..70ed549a 100644
--- a/tests/test_runtime_fixes.py
+++ b/tests/test_runtime_fixes.py
@@ -9,11 +9,9 @@ import os
import subprocess
import sys
from contextlib import nullcontext
-from types import SimpleNamespace
from unittest.mock import patch
import torch
-import torch.nn as nn
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -21,7 +19,7 @@ from laya.agent import ( # noqa: E402
MPS_AMP_MIN_ROWS_DEFAULT, Agent, _BATCH_AUTOCAST_CACHE, _amp_context, _cpu_amp_dtype,
_cuda_amp_dtype, _mps_amp_min_rows,
)
-from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402
+from laya.common import build_sequence, serialize_state # noqa: E402
PASS, FAIL = [], []
@@ -135,6 +133,32 @@ check("build_sequence/option segments stay <= 49 tokens",
# ------------------------------------------------------------------ autocast
stage("autocast")
+# The real bf16 forward -- this model's own ops, not a stand-in -- and the only place in this file
+# that executes one. It runs in a child process rather than here, because on some machines that
+# forward is a native fault and a fault is not an exception: it kills the interpreter, so no
+# `except` sees it and no `check` records it. `tests (windows)` has taken this suite down with
+# `exit 3221225501 (0xC000001D)`, STATUS_ILLEGAL_INSTRUCTION, about three runs in thirty-five; the
+# stage markers narrowed it to this section, which then printed `[stage] autocast` and died. It is
+# the class of issue #1065, where the fbgemm quantization engine raised SIGILL on a CPU with SSE4.2
+# and no AVX2 -- oneDNN chooses a kernel for an instruction set the CPU does not have -- and it is
+# intermittent on one runner label because the hosted fleet is not one CPU model.
+#
+# A capability probe built from `nn.Linear(16, 16)` was not enough to predict it: oneDNN chooses
+# per op and per shape, so the toy GEMM ran on a runner where this model's ops did not. The probe
+# is therefore the assertion itself. The child runs both legs and prints its numbers; this process
+# only ever reads them, and so never touches a bf16 kernel. If the child dies, the section says so
+# with the capability torch reports instead of taking the suite down; if it survives, the numbers
+# are checked for real. laya's own behaviour is unchanged either way: `LAYA_CPU_AMP` is opt-in, and
+# `_cpu_amp_dtype` documents that it is for hardware with native BF16 instructions.
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_AMP_PROBE = r'''
+import torch
+import torch.nn as nn
+from types import SimpleNamespace
+
+from laya.common import DecisionModel
+
+
class DummyEnc(nn.Module):
def __init__(self, d=16):
super().__init__()
@@ -151,48 +175,40 @@ am = torch.ones_like(iid)
mp = torch.tensor([[1, 3]])
mm = torch.ones_like(mp, dtype=torch.bool)
qt = torch.tensor([0])
-
-
-# Whether this CPU can execute a bf16 autocast forward at all, asked in a child process, because
-# the answer on some machines is a native fault that no `except` can see. `tests (windows)` has
-# killed this suite with `0xC000001D` (STATUS_ILLEGAL_INSTRUCTION) about three runs in
-# thirty-five, and the stage markers added for it narrowed the fault to this section: the lane
-# printed `[stage] state reused once` and `[stage] autocast` and then died. It is the same class
-# as issue #1065, where the fbgemm quantization engine raised SIGILL on a CPU with SSE4.2 and no
-# AVX2 -- oneDNN picks a kernel for an instruction set the CPU does not have. It is intermittent
-# on one runner label because the hosted fleet is not one CPU model.
-#
-# So the three checks below run only where the forward is executable, and where it is not the
-# suite says so with the capability torch reports instead of taking the process down. laya's own
-# behaviour is unchanged either way: `LAYA_CPU_AMP` is opt-in, and `_cpu_amp_dtype` documents that
-# it is for hardware with native BF16 instructions.
-_AMP_PROBE = (
- "import torch, torch.nn as nn\n"
- "m = nn.Linear(16, 16)\n"
- "x = torch.ones(1, 16)\n"
- "with torch.no_grad(), torch.autocast(device_type='cpu', dtype=torch.bfloat16, enabled=True):\n"
- " m(x)\n"
- "print('ok')\n"
-)
-_probe = subprocess.run([sys.executable, "-c", _AMP_PROBE], capture_output=True, text=True)
-_amp_runs = _probe.returncode == 0 and _probe.stdout.strip() == "ok"
+with torch.no_grad():
+ logits32, act32 = model(iid, am, mp, mm, qt)
+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16, enabled=True):
+ logits16, act16 = model(iid, am, mp, mm, qt)
+finite = bool(torch.isfinite(logits16).all() and torch.isfinite(act16).all())
+delta = float((logits32 - logits16).abs().max())
+# shapes, then a 0/1 for finite, then the fp32/bf16 gap: parsed, never re-derived, by the parent
+print("ok", "x".join(str(n) for n in logits16.shape), "x".join(str(n) for n in logits32.shape),
+ int(finite), delta)
+'''
+try:
+ # cwd is the repo root, which `python -c` puts first on `sys.path`, so `laya` imports as usual.
+ # The timeout only bounds a hang; a faulting child exits at once with the status above.
+ _probe = subprocess.run([sys.executable, "-c", _AMP_PROBE], capture_output=True, text=True,
+ cwd=_ROOT, timeout=600)
+except subprocess.TimeoutExpired: # pragma: no cover
+ _probe = None
+_stdout = "" if _probe is None else (_probe.stdout or "")
+_lines = [ln.split() for ln in _stdout.splitlines() if ln.startswith("ok ")]
+_words = _lines[-1] if (_probe is not None and _probe.returncode == 0 and _lines) else []
+_amp_runs = len(_words) == 5
if not _amp_runs:
try:
_capability = torch.backends.cpu.get_cpu_capability()
except Exception as exc: # pragma: no cover
_capability = "unknown (%s)" % type(exc).__name__
stage("autocast: SKIPPED, this CPU cannot execute a bf16 autocast forward "
- "(capability %s, child exit %s)" % (_capability, _probe.returncode))
+ "(capability %s, child exit %s)"
+ % (_capability, "timeout" if _probe is None else _probe.returncode))
PASS.append("autocast/skipped: bf16 CPU autocast is not executable here")
else:
- with torch.no_grad():
- logits32, act32 = model(iid, am, mp, mm, qt)
- with torch.autocast(device_type="cpu", dtype=torch.bfloat16, enabled=True):
- logits16, act16 = model(iid, am, mp, mm, qt)
- check("autocast/shapes match", tuple(logits16.shape), tuple(logits32.shape))
- check("autocast/finite",
- bool(torch.isfinite(logits16).all() and torch.isfinite(act16).all()), True)
- check("autocast/close to fp32", float((logits32 - logits16).abs().max()) < 0.5, True)
+ check("autocast/shapes match", _words[1], _words[2])
+ check("autocast/finite", _words[3] == "1", True)
+ check("autocast/close to fp32", float(_words[4]) < 0.5, True)
calls = {"n": 0}