Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): put the CPU autocast vocabulary in a function the tests can reach

Aashish60b5c860535b
diff --git a/laya/agent.py b/laya/agent.py
index c98e6859..cd039909 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -225,8 +225,10 @@ def _mps_amp_min_rows() -> int:
def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:
"""Autocast dtype on CUDA at compute capability >= 8: the checkpoint's `amp_dtype` (bf16 for the
- shipped checkpoints), or LAYA_CUDA_AMP=fp16|bf16 when set. fp16 stays 2-10x closer to the fp32
- forward than bf16 on every shipped checkpoint at the same speed; anything else is ignored."""
+ shipped checkpoints), or LAYA_CUDA_AMP=fp16|float16|bf16|bfloat16 when set. fp16 stays 2-10x
+ closer to the fp32 forward than bf16 on every shipped checkpoint at the same speed; anything
+ else is ignored. The long spellings count because they are what `agent.dtype` reports back, so
+ the precision a user reads off one response is one they can ask for on the next."""
raw = os.environ.get("LAYA_CUDA_AMP", "").lower()
if raw in ("fp16", "float16"):
return torch.float16
@@ -234,6 +236,16 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:
return torch.bfloat16
return amp_dtype(checkpoint_default)
+
+def _cpu_amp_dtype() -> Optional[torch.dtype]:
+ """Autocast dtype on CPU: bf16 when LAYA_CPU_AMP=bf16|bfloat16, None to leave the forward in
+ fp32. No fp16 spelling is offered here, because CPU autocast has no fp16 fast path that beats
+ fp32 -- bf16 is the only reduced precision worth asking for, and only on hardware with native
+ BF16 instructions, which is why it stays opt-in. Anything else is ignored."""
+ if os.environ.get("LAYA_CPU_AMP", "").lower() in ("bf16", "bfloat16"):
+ return torch.bfloat16
+ return None
+
def _option_count(qdef: Dict) -> int:
"""How many options a validated question definition renders to.
@@ -536,8 +548,9 @@ class Agent(HookRegistry):
self.load_calibration(calibration)
# Autocast policy. CUDA, MPS and XPU all support fp16/bf16 autocast and the shipped
# checkpoints are trained in reduced precision; on CUDA the checkpoint's `amp_dtype`
- # (bf16) is the default and LAYA_CUDA_AMP=fp16|bf16 overrides it. CPU bf16 is only a win
- # on hardware with native BF16, so it stays opt-in via LAYA_CPU_AMP=bf16. MPS fp16 is slower than fp32 on
+ # (bf16) is the default and _cuda_amp_dtype's LAYA_CUDA_AMP override wins over it. CPU
+ # bf16 is only a win on hardware with native BF16, so it stays opt-in through
+ # _cpu_amp_dtype. MPS fp16 is slower than fp32 on
# a single small row (autocast overhead dominates) and only wins once the batch has
# several rows, so it is gated per call by `mps_amp_min_rows` (default 5, override with
# LAYA_MPS_AMP_MIN_ROWS) rather than enabled unconditionally. XPU autocast supports
@@ -563,9 +576,10 @@ class Agent(HookRegistry):
self.amp_enabled = True
self.dtype = torch.bfloat16
elif self.device.type == "cpu":
- if os.environ.get("LAYA_CPU_AMP", "").lower() in ("bf16", "bfloat16"):
+ cpu_amp = _cpu_amp_dtype()
+ if cpu_amp is not None:
self.amp_enabled = True
- self.dtype = torch.bfloat16
+ self.dtype = cpu_amp
self._fast = None
diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py
index 6da1bb6b..4f06a87f 100644
--- a/tests/test_runtime_fixes.py
+++ b/tests/test_runtime_fixes.py
@@ -17,7 +17,8 @@ import torch.nn as nn
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya.agent import ( # noqa: E402
- MPS_AMP_MIN_ROWS_DEFAULT, Agent, _BATCH_AUTOCAST_CACHE, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows,
+ MPS_AMP_MIN_ROWS_DEFAULT, Agent, _BATCH_AUTOCAST_CACHE, _amp_context, _cpu_amp_dtype,
+ _cuda_amp_dtype, _mps_amp_min_rows,
)
from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402
@@ -263,8 +264,31 @@ os.environ["LAYA_CUDA_AMP"] = "BF16"
check("cuda-amp/env bf16 overrides an fp16 checkpoint", _cuda_amp_dtype("fp16"), torch.bfloat16)
os.environ["LAYA_CUDA_AMP"] = "int8"
check("cuda-amp/env invalid falls back to the checkpoint", _cuda_amp_dtype("bf16"), torch.bfloat16)
+# `agent.dtype` reports float16 and bfloat16, so those are the spellings a caller can read off one
+# response and ask for on the next. laya/agent.py has accepted both from the start; both prose
+# sites that listed the vocabulary described them as inert, and tests/test_env_docs.py now holds
+# every page that names a dtype to the tuples above.
+os.environ["LAYA_CUDA_AMP"] = "float16"
+check("cuda-amp/env float16 is the same ask as fp16", _cuda_amp_dtype("bf16"), torch.float16)
+os.environ["LAYA_CUDA_AMP"] = "BFloat16"
+check("cuda-amp/env bfloat16 is the same ask as bf16", _cuda_amp_dtype("fp16"), torch.bfloat16)
del os.environ["LAYA_CUDA_AMP"]
+# CPU has its own vocabulary and it is the narrower one: bf16 only. No arm reached this comparison
+# before -- the agents in this file hand-set `dtype` and `amp_enabled` through `_bare_agent`, so
+# the value the documentation promises was never the value anything checked.
+os.environ.pop("LAYA_CPU_AMP", None)
+check("cpu-amp/unset leaves the forward fp32", _cpu_amp_dtype(), None)
+os.environ["LAYA_CPU_AMP"] = "bf16"
+check("cpu-amp/env bf16 opts in", _cpu_amp_dtype(), torch.bfloat16)
+os.environ["LAYA_CPU_AMP"] = "BFloat16"
+check("cpu-amp/env bfloat16 opts in", _cpu_amp_dtype(), torch.bfloat16)
+os.environ["LAYA_CPU_AMP"] = "fp16"
+check("cpu-amp/env fp16 is not offered on this device", _cpu_amp_dtype(), None)
+os.environ["LAYA_CPU_AMP"] = "int8"
+check("cpu-amp/env invalid leaves the forward fp32", _cpu_amp_dtype(), None)
+del os.environ["LAYA_CPU_AMP"]
+
# ------------------------------------------------------------------ amp context shape
# A disabled gate must never construct torch.autocast: on torch builds without an MPS