The best ChatGPT that $100 can buy.
Python
mqa -> gqa to reduce confusion
Andrej Karpathybc1fca39f330
diff --git a/nanochat/gpt.py b/nanochat/gpt.pyindex b640f1e..8b220c3 100644--- a/nanochat/gpt.py+++ b/nanochat/gpt.py@@ -8,7 +8,7 @@ Notable features:- norm after token embedding- no learnable params in rmsnorm- no bias in linear layers-- Multi-Query Attention (MQA) support for more efficient inference+- Group-Query Attention (GQA) support for more efficient inference"""import math@@ -29,7 +29,7 @@ class GPTConfig:vocab_size: int = 50304n_layer: int = 12n_head: int = 6 # number of query heads- n_kv_head: int = 6 # number of key/value heads (MQA)+ n_kv_head: int = 6 # number of key/value heads (GQA)n_embd: int = 768