Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

avoid overflow

eugene4e4eb6865511
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 54048bbfc..0f8b2a15a 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -19,7 +19,10 @@ inline T gelu_approx(T x) {
constexpr float k0 = 0.044715f;
constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
float xf = float(x);
- float yf = 0.5f * xf * (1.0f + tanh(k1 * (xf + k0 * xf * xf * xf)));
+ float t = k1 * (xf + k0 * xf * xf * xf);
+ // Avoid NaNs from fast `tanh` implementations that may overflow internally.
+ t = clamp(t, -20.0f, 20.0f);
+ float yf = 0.5f * xf * (1.0f + metal::precise::tanh(t));
return T(yf);
}
@@ -47,7 +50,8 @@ template <typename T>
T up = fused_up[base + j];
T gate = fused_up[base + H + j];
T g = activate(gate, act_type);
- hidden[row * H + j] = up * g;
+ float out_f = float(up) * float(g);
+ hidden[row * H + j] = T(out_f);
}
// Explicit instantiations with stable host names