The best ChatGPT that $100 can buy.
Python
remove spurious cast, gets compiled away anyway but it's confusing people
Andrej Karpathye1770a3061df
diff --git a/nanochat/gpt.py b/nanochat/gpt.pyindex 9a80c7c..69899ee 100644--- a/nanochat/gpt.py+++ b/nanochat/gpt.py@@ -41,12 +41,10 @@ def norm(x):def apply_rotary_emb(x, cos, sin):assert x.ndim == 4 # multihead attentiond = x.shape[3] // 2- x1, x2 = x[..., :d], x[..., d:] # split up last time into two halves+ x1, x2 = x[..., :d], x[..., d:] # split up last dim into two halvesy1 = x1 * cos + x2 * sin # rotate pairs of dimsy2 = x1 * (-sin) + x2 * cos- out = torch.cat([y1, y2], 3) # re-assemble- out = out.to(x.dtype) # ensure input/output dtypes match- return out+ return torch.cat([y1, y2], 3)class CausalSelfAttention(nn.Module):def __init__(self, config, layer_idx):