Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(agent): LAYA_CUDA_AMP=fp16|bf16 overrides the CUDA autocast dtype

Bhushan Kinge2782a128dff0
diff --git a/README.md b/README.md
index 0560d3e6..6a7a3f85 100644
--- a/README.md
+++ b/README.md
@@ -586,6 +586,8 @@ else:
A threshold is a policy you choose from measured accuracy at that coverage on your data, not a property of the model. Both checkpoints are over-confident as shipped and `laya-multilingual` has no fitted temperatures at all, so fit them before relying on these numbers — see [Calibration](#calibration) above, and the [fine-tuning notebook](notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb) for the fitting loop itself. Then pick the point where the errors you accept are ones you can live with. Confidence orders decisions; it does not establish that a decision is correct.
+A threshold also depends on the autocast dtype. On CUDA at compute capability 8 or above the runtime uses the checkpoint's `amp_dtype`, which is bf16 for all three shipped checkpoints. On the fixed set from `benchmarks/parity_fast.py` (60 states, 288 questions per checkpoint, RTX 2000 Ada) bf16 moves a probability by up to 0.073 against the fp32 forward and flips 3 of 864 argmaxes across the three checkpoints; fp16 stays within 0.019 and flips none, at the same latency. `LAYA_CUDA_AMP=fp16` selects fp16 and `LAYA_CUDA_AMP=bf16` selects bf16 (`LAYA_CPU_AMP=bf16` is the CPU counterpart). Fit and measure a threshold in the dtype you serve with.
+
---
## Prediction Hooks
diff --git a/laya/agent.py b/laya/agent.py
index a76041f7..880b23cd 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -192,6 +192,18 @@ def _mps_amp_min_rows() -> int:
return MPS_AMP_MIN_ROWS_DEFAULT
+def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:
+ """Autocast dtype on CUDA at compute capability >= 8: the checkpoint's `amp_dtype` (bf16 for the
+ shipped checkpoints), or LAYA_CUDA_AMP=fp16|bf16 when set. fp16 stays 2-10x closer to the fp32
+ forward than bf16 on every shipped checkpoint at the same speed; anything else is ignored."""
+ raw = os.environ.get("LAYA_CUDA_AMP", "").lower()
+ if raw in ("fp16", "float16"):
+ return torch.float16
+ if raw in ("bf16", "bfloat16"):
+ return torch.bfloat16
+ return amp_dtype(checkpoint_default)
+
+
class Agent(HookRegistry):
"""System 1 decision model runtime: fast, non-autoregressive, calibrated decisions."""
@@ -384,8 +396,9 @@ class Agent(HookRegistry):
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
RuntimeWarning, stacklevel=2)
# Autocast policy. CUDA and MPS both support fp16/bf16 autocast and the shipped
- # checkpoints are trained in reduced precision. CPU bf16 is only a win on hardware with
- # native BF16, so it stays opt-in via LAYA_CPU_AMP=bf16. MPS fp16 is slower than fp32 on
+ # checkpoints are trained in reduced precision; on CUDA the checkpoint's `amp_dtype`
+ # (bf16) is the default and LAYA_CUDA_AMP=fp16|bf16 overrides it. CPU bf16 is only a win
+ # on hardware with native BF16, so it stays opt-in via LAYA_CPU_AMP=bf16. MPS fp16 is slower than fp32 on
# a single small row (autocast overhead dominates) and only wins once the batch has
# several rows, so it is gated per call by `mps_amp_min_rows` (default 5, override with
# LAYA_MPS_AMP_MIN_ROWS) rather than enabled unconditionally.
@@ -397,7 +410,7 @@ class Agent(HookRegistry):
if torch.cuda.get_device_capability(self.device)[0] < 8:
self.dtype = torch.float16
else:
- self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))
+ self.dtype = _cuda_amp_dtype(self.cfg.get("amp_dtype", "fp16"))
elif self.device.type == "mps":
self.amp_enabled = True
self.dtype = torch.float16
diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py
index 5875ee86..a769a2db 100644
--- a/tests/test_runtime_fixes.py
+++ b/tests/test_runtime_fixes.py
@@ -15,7 +15,7 @@ import torch.nn as nn
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _mps_amp_min_rows # noqa: E402
+from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _cuda_amp_dtype, _mps_amp_min_rows # noqa: E402
from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402
PASS, FAIL = [], []
@@ -210,6 +210,17 @@ check("mps-gate/env invalid falls back", _mps_amp_min_rows(), MPS_AMP_MIN_ROWS_D
del os.environ["LAYA_MPS_AMP_MIN_ROWS"]
check("mps-gate/env default", _mps_amp_min_rows(), MPS_AMP_MIN_ROWS_DEFAULT)
+os.environ.pop("LAYA_CUDA_AMP", None)
+check("cuda-amp/checkpoint default wins when unset", _cuda_amp_dtype("bf16"), torch.bfloat16)
+check("cuda-amp/no checkpoint value means fp16", _cuda_amp_dtype(None), torch.float16)
+os.environ["LAYA_CUDA_AMP"] = "fp16"
+check("cuda-amp/env fp16 overrides a bf16 checkpoint", _cuda_amp_dtype("bf16"), torch.float16)
+os.environ["LAYA_CUDA_AMP"] = "BF16"
+check("cuda-amp/env bf16 overrides an fp16 checkpoint", _cuda_amp_dtype("fp16"), torch.bfloat16)
+os.environ["LAYA_CUDA_AMP"] = "int8"
+check("cuda-amp/env invalid falls back to the checkpoint", _cuda_amp_dtype("bf16"), torch.bfloat16)
+del os.environ["LAYA_CUDA_AMP"]
+
# ------------------------------------------------------------------ amp context shape
# A disabled gate must never construct torch.autocast: on torch builds without an MPS