The best ChatGPT that $100 can buy.
Python
Merge branch 'master' into fix/kernel
svlandega14c399576b3
diff --cc nanochat/gpt.pyindex 39c1d60,5e99c73..3775bc6--- a/nanochat/gpt.py+++ b/nanochat/gpt.py@@@ -19,10 -19,10 +19,10 @@@ import torcimport torch.nn as nnimport torch.nn.functional as F- from nanochat.common import get_dist_info, print0+ from nanochat.common import get_dist_info, print0, COMPUTE_DTYPEfrom nanochat.optim import MuonAdamW, DistMuonAdamW-# Our custom Flash Attention module that automatically uses FA3 on Hopper+ and SDPA fallback elsewhere+# Our custom Flash Attention module that automatically uses FA3 when compatible and SDPA fallback otherwisefrom nanochat.flash_attention import flash_attn@dataclass@@@ -92,8 -98,10 +98,10 @@@ class CausalSelfAttention(nn.Module)cos, sin = cos_sinq, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)q, k = norm(q), norm(k) # QK norm+ q = q * 1.15 # sharper attention (split scale between Q and K), TODO think through better+ k = k * 1.15- # Flash Attention (FA3 on Hopper+, PyTorch SDPA fallback elsewhere)+ # Flash Attention (FA3 or SDPA fallback)# window_size is (left, right) tuple: (N, 0) for causal, (-1, 0) for full contextif kv_cache is None:# Training: causal attention with optional sliding windowdiff --cc scripts/base_train.pyindex 1d725eb,cfbfe28..2b564ea--- a/scripts/base_train.py+++ b/scripts/base_train.py@@@ -101,11 -100,16 +100,16 @@@ use_dummy_wandb = args.run == "dummy" owandb_run = DummyWandb() if use_dummy_wandb else wandb.init(project="nanochat", name=args.run, config=user_config)# Flash Attention status- if HAS_FA3:+ from nanochat.flash_attention import USE_FA3+ using_fa3 = USE_FA3+ if using_fa3:- print0("✓ Using Flash Attention 3 (Hopper GPU detected), efficient, new and awesome.")+ print0("✓ Using Flash Attention 3: efficient, new and awesome.")else:print0("!" * 80)- print0("WARNING: Flash Attention 3 not available, using PyTorch SDPA fallback")+ if HAS_FA3 and COMPUTE_DTYPE != torch.bfloat16:+ print0(f"WARNING: Flash Attention 3 only supports bf16, but COMPUTE_DTYPE={COMPUTE_DTYPE}. Using PyTorch SDPA fallback")+ else:+ print0("WARNING: Flash Attention 3 not available, using PyTorch SDPA fallback")print0("WARNING: Training will be less efficient without FA3")if args.window_pattern != "L":print0(f"WARNING: SDPA has no support for sliding window attention (window_pattern='{args.window_pattern}'). Your GPU utilization will be terrible.")