Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): put the CPU autocast vocabulary in a function the tests can reach
Aashish60b5c860535b
diff --git a/laya/agent.py b/laya/agent.pyindex c98e6859..cd039909 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -225,8 +225,10 @@ def _mps_amp_min_rows() -> int:def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:"""Autocast dtype on CUDA at compute capability >= 8: the checkpoint's `amp_dtype` (bf16 for the- shipped checkpoints), or LAYA_CUDA_AMP=fp16|bf16 when set. fp16 stays 2-10x closer to the fp32- forward than bf16 on every shipped checkpoint at the same speed; anything else is ignored."""+ shipped checkpoints), or LAYA_CUDA_AMP=fp16|float16|bf16|bfloat16 when set. fp16 stays 2-10x+ closer to the fp32 forward than bf16 on every shipped checkpoint at the same speed; anything+ else is ignored. The long spellings count because they are what `agent.dtype` reports back, so+ the precision a user reads off one response is one they can ask for on the next."""raw = os.environ.get("LAYA_CUDA_AMP", "").lower()if raw in ("fp16", "float16"):return torch.float16@@ -234,6 +236,16 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:return torch.bfloat16return amp_dtype(checkpoint_default)++def _cpu_amp_dtype() -> Optional[torch.dtype]:+ """Autocast dtype on CPU: bf16 when LAYA_CPU_AMP=bf16|bfloat16, None to leave the forward in+ fp32. No fp16 spelling is offered here, because CPU autocast has no fp16 fast path that beats+ fp32 -- bf16 is the only reduced precision worth asking for, and only on hardware with native+ BF16 instructions, which is why it stays opt-in. Anything else is ignored."""+ if os.environ.get("LAYA_CPU_AMP", "").lower() in ("bf16", "bfloat16"):+ return torch.bfloat16+ return None+def _option_count(qdef: Dict) -> int:"""How many options a validated question definition renders to.@@ -536,8 +548,9 @@ class Agent(HookRegistry):self.load_calibration(calibration)# Autocast policy. CUDA, MPS and XPU all support fp16/bf16 autocast and the shipped# checkpoints are trained in reduced precision; on CUDA the checkpoint's `amp_dtype`- # (bf16) is the default and LAYA_CUDA_AMP=fp16|bf16 overrides it. CPU bf16 is only a win- # on hardware with native BF16, so it stays opt-in via LAYA_CPU_AMP=bf16. MPS fp16 is slower than fp32 on+ # (bf16) is the default and _cuda_amp_dtype's LAYA_CUDA_AMP override wins over it. CPU+ # bf16 is only a win on hardware with native BF16, so it stays opt-in through+ # _cpu_amp_dtype. MPS fp16 is slower than fp32 on# a single small row (autocast overhead dominates) and only wins once the batch has# several rows, so it is gated per call by `mps_amp_min_rows` (default 5, override with# LAYA_MPS_AMP_MIN_ROWS) rather than enabled unconditionally. XPU autocast supports@@ -563,9 +576,10 @@ class Agent(HookRegistry):self.amp_enabled = Trueself.dtype = torch.bfloat16elif self.device.type == "cpu":- if os.environ.get("LAYA_CPU_AMP", "").lower() in ("bf16", "bfloat16"):+ cpu_amp = _cpu_amp_dtype()+ if cpu_amp is not None:self.amp_enabled = True- self.dtype = torch.bfloat16+ self.dtype = cpu_ampself._fast = Nonediff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.pyindex 6da1bb6b..4f06a87f 100644--- a/tests/test_runtime_fixes.py+++ b/tests/test_runtime_fixes.py@@ -17,7 +17,8 @@ import torch.nn as nnsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))from laya.agent import ( # noqa: E402- MPS_AMP_MIN_ROWS_DEFAULT, Agent, _BATCH_AUTOCAST_CACHE, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows,+ MPS_AMP_MIN_ROWS_DEFAULT, Agent, _BATCH_AUTOCAST_CACHE, _amp_context, _cpu_amp_dtype,+ _cuda_amp_dtype, _mps_amp_min_rows,)from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402@@ -263,8 +264,31 @@ os.environ["LAYA_CUDA_AMP"] = "BF16"check("cuda-amp/env bf16 overrides an fp16 checkpoint", _cuda_amp_dtype("fp16"), torch.bfloat16)os.environ["LAYA_CUDA_AMP"] = "int8"check("cuda-amp/env invalid falls back to the checkpoint", _cuda_amp_dtype("bf16"), torch.bfloat16)+# `agent.dtype` reports float16 and bfloat16, so those are the spellings a caller can read off one+# response and ask for on the next. laya/agent.py has accepted both from the start; both prose+# sites that listed the vocabulary described them as inert, and tests/test_env_docs.py now holds+# every page that names a dtype to the tuples above.+os.environ["LAYA_CUDA_AMP"] = "float16"+check("cuda-amp/env float16 is the same ask as fp16", _cuda_amp_dtype("bf16"), torch.float16)+os.environ["LAYA_CUDA_AMP"] = "BFloat16"+check("cuda-amp/env bfloat16 is the same ask as bf16", _cuda_amp_dtype("fp16"), torch.bfloat16)del os.environ["LAYA_CUDA_AMP"]+# CPU has its own vocabulary and it is the narrower one: bf16 only. No arm reached this comparison+# before -- the agents in this file hand-set `dtype` and `amp_enabled` through `_bare_agent`, so+# the value the documentation promises was never the value anything checked.+os.environ.pop("LAYA_CPU_AMP", None)+check("cpu-amp/unset leaves the forward fp32", _cpu_amp_dtype(), None)+os.environ["LAYA_CPU_AMP"] = "bf16"+check("cpu-amp/env bf16 opts in", _cpu_amp_dtype(), torch.bfloat16)+os.environ["LAYA_CPU_AMP"] = "BFloat16"+check("cpu-amp/env bfloat16 opts in", _cpu_amp_dtype(), torch.bfloat16)+os.environ["LAYA_CPU_AMP"] = "fp16"+check("cpu-amp/env fp16 is not offered on this device", _cpu_amp_dtype(), None)+os.environ["LAYA_CPU_AMP"] = "int8"+check("cpu-amp/env invalid leaves the forward fp32", _cpu_amp_dtype(), None)+del os.environ["LAYA_CPU_AMP"]+# ------------------------------------------------------------------ amp context shape# A disabled gate must never construct torch.autocast: on torch builds without an MPS