Skip to content

Distribute and run LLMs with a single file.

Cherry pick gemma fix from upstream

Justine Tunneyb2f240ee46fd
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 1667cd56..83932a5e 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -10277,7 +10277,10 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty
return std::make_pair(i_layer, n_layer);
};
- if (name == tn(LLM_TENSOR_OUTPUT, "weight")) {
+ // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings
+ // with the quantization of the output tensor
+ if (name == tn(LLM_TENSOR_OUTPUT, "weight") ||
+ (LLM_TENSOR_NAMES.at(arch).find(LLM_TENSOR_OUTPUT) == LLM_TENSOR_NAMES.at(arch).end() && name == "token_embd.weight")) {
int nx = tensor->ne[0];
if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) {
new_type = GGML_TYPE_Q8_0;