The best ChatGPT that $100 can buy.
Python
allocate the inference KV cache in COMPUTE_DTYPE instead of hardcoding cuda->bf16, fixing bf16 inference (e.g. NANOCHAT_DTYPE=bfloat16 on MPS) and deleting an old hack. Thank you @mparrett #795
Andrej Karpathyca366eb18dd2
diff --git a/nanochat/engine.py b/nanochat/engine.pyindex aa2e6a9..9680d83 100644--- a/nanochat/engine.py+++ b/nanochat/engine.py@@ -17,7 +17,7 @@ import signalimport warningsfrom contextlib import contextmanagerfrom collections import deque-from nanochat.common import compute_init, autodetect_device_type+from nanochat.common import compute_init, autodetect_device_type, COMPUTE_DTYPEfrom nanochat.checkpoint_manager import load_model# -----------------------------------------------------------------------------@@ -177,13 +177,8 @@ class Engine:"""Same as generate, but does single prefill and then clones the KV cache."""assert isinstance(tokens, list) and isinstance(tokens[0], int), "expecting list of ints"device = self.model.get_device()- # NOTE: setting the dtype here and in this way is an ugly hack.- # Currently the repo assumes that cuda -> bfloat16 and everything else -> float32.- # We need to know the dtype here to call __init__ on KVCache and pre-allocate its tensors.- # As a quick hack, we're making generate() function inherit and know about this repo-wise assumption.- # I think there has to be a bigger refactor to deal with device/dtype tracking across the codebase.- # In particular, the KVCache should allocate its tensors lazily- dtype = torch.bfloat16 if device.type == "cuda" else torch.float32+ # Allocate the KV cache in the compute dtype so it matches what the forward pass emits+ dtype = COMPUTE_DTYPErng = torch.Generator(device=device)rng.manual_seed(seed)