Skip to content

The best ChatGPT that $100 can buy.

Python

Merge branch 'master' into fix/kernel

svlandega14c399576b3
diff --cc nanochat/gpt.py
index 39c1d60,5e99c73..3775bc6
--- a/nanochat/gpt.py
+++ b/nanochat/gpt.py
@@@ -19,10 -19,10 +19,10 @@@ import torc
import torch.nn as nn
import torch.nn.functional as F
- from nanochat.common import get_dist_info, print0
+ from nanochat.common import get_dist_info, print0, COMPUTE_DTYPE
from nanochat.optim import MuonAdamW, DistMuonAdamW
-# Our custom Flash Attention module that automatically uses FA3 on Hopper+ and SDPA fallback elsewhere
+# Our custom Flash Attention module that automatically uses FA3 when compatible and SDPA fallback otherwise
from nanochat.flash_attention import flash_attn
@dataclass
@@@ -92,8 -98,10 +98,10 @@@ class CausalSelfAttention(nn.Module)
cos, sin = cos_sin
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
q, k = norm(q), norm(k) # QK norm
+ q = q * 1.15 # sharper attention (split scale between Q and K), TODO think through better
+ k = k * 1.15
- # Flash Attention (FA3 on Hopper+, PyTorch SDPA fallback elsewhere)
+ # Flash Attention (FA3 or SDPA fallback)
# window_size is (left, right) tuple: (N, 0) for causal, (-1, 0) for full context
if kv_cache is None:
# Training: causal attention with optional sliding window
diff --cc scripts/base_train.py
index 1d725eb,cfbfe28..2b564ea
--- a/scripts/base_train.py
+++ b/scripts/base_train.py
@@@ -101,11 -100,16 +100,16 @@@ use_dummy_wandb = args.run == "dummy" o
wandb_run = DummyWandb() if use_dummy_wandb else wandb.init(project="nanochat", name=args.run, config=user_config)
# Flash Attention status
- if HAS_FA3:
+ from nanochat.flash_attention import USE_FA3
+ using_fa3 = USE_FA3
+ if using_fa3:
- print0("✓ Using Flash Attention 3 (Hopper GPU detected), efficient, new and awesome.")
+ print0("✓ Using Flash Attention 3: efficient, new and awesome.")
else:
print0("!" * 80)
- print0("WARNING: Flash Attention 3 not available, using PyTorch SDPA fallback")
+ if HAS_FA3 and COMPUTE_DTYPE != torch.bfloat16:
+ print0(f"WARNING: Flash Attention 3 only supports bf16, but COMPUTE_DTYPE={COMPUTE_DTYPE}. Using PyTorch SDPA fallback")
+ else:
+ print0("WARNING: Flash Attention 3 not available, using PyTorch SDPA fallback")
print0("WARNING: Training will be less efficient without FA3")
if args.window_pattern != "L":
print0(f"WARNING: SDPA has no support for sliding window attention (window_pattern='{args.window_pattern}'). Your GPU utilization will be terrible.")