A high-performance inference engine for AI models
avoid overflow
eugene4e4eb6865511
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 54048bbfc..0f8b2a15a 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -19,7 +19,10 @@ inline T gelu_approx(T x) {constexpr float k0 = 0.044715f;constexpr float k1 = 0.7978845608f; // sqrt(2/pi)float xf = float(x);- float yf = 0.5f * xf * (1.0f + tanh(k1 * (xf + k0 * xf * xf * xf)));+ float t = k1 * (xf + k0 * xf * xf * xf);+ // Avoid NaNs from fast `tanh` implementations that may overflow internally.+ t = clamp(t, -20.0f, 20.0f);+ float yf = 0.5f * xf * (1.0f + metal::precise::tanh(t));return T(yf);}@@ -47,7 +50,8 @@ template <typename T>T up = fused_up[base + j];T gate = fused_up[base + H + j];T g = activate(gate, act_type);- hidden[row * H + j] = up * g;+ float out_f = float(up) * float(g);+ hidden[row * H + j] = T(out_f);}// Explicit instantiations with stable host names