Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Fix block-size assumption (#935)
Davide Eynarde7e6796f9f4a
diff --git a/llamafile/iqk_mul_mat.inc b/llamafile/iqk_mul_mat.incindex 8043cec6..dbcbba78 100644--- a/llamafile/iqk_mul_mat.inc+++ b/llamafile/iqk_mul_mat.inc@@ -1423,8 +1423,6 @@ template <typename Dequantizer> void MulMat::set_functions(MulMat& m) {bool MulMat::set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8, int) {- row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00);-switch (typeA) {case GGML_TYPE_Q2_K:assert (ne00 % QK_K == 0);@@ -1454,27 +1452,29 @@ bool MulMat::set_mul_mat(int typeA, int ne00, MulMat& mm, int& row_size_q8, int)assert (ne00 % QK4_0 == 0);MulMat::set_functions<Q4_0_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);- break;+ return true;case GGML_TYPE_Q4_1:assert (ne00 % QK4_1 == 0);MulMat::set_functions<Q4_1_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);- break;+ return true;case GGML_TYPE_Q5_0:assert (ne00 % QK5_0 == 0);MulMat::set_functions<Q5_0_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);- break;+ return true;case GGML_TYPE_Q5_1:assert (ne00 % QK5_1 == 0);MulMat::set_functions<Q5_1_Unpacker>(mm);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);- break;+ return true;default:return false;}+ // Only K-quant types reach here (they break out of the switch above)+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00);return true;}@@ -3028,7 +3028,6 @@ template <typename Dequantizer> void MulMat::set_functions(MulMat& m) {}bool MulMat::set_mul_mat(int typeA, int ne00, MulMat& m, int& row_size_q8, int Ny) {- row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00);(void)Ny;// Uncommenting out this would disable iqk_mul_mat for matrix x vector multiplications.@@ -3072,26 +3071,29 @@ bool MulMat::set_mul_mat(int typeA, int ne00, MulMat& m, int& row_size_q8, int Ncase GGML_TYPE_Q4_0:MulMat::set_functions<DequantizerQ40>(m);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);- break;+ return true;case GGML_TYPE_Q4_1:MulMat::set_functions<DequantizerQ41>(m);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);- break;+ return true;case GGML_TYPE_Q5_0:MulMat::set_functions<DequantizerQ50>(m);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);- break;+ return true;case GGML_TYPE_Q5_1:MulMat::set_functions<DequantizerQ51>(m);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_1, ne00);- break;+ return true;case GGML_TYPE_Q8_0:MulMat::set_functions<DequantizerQ80>(m);row_size_q8 = ggml_row_size(GGML_TYPE_Q8_0, ne00);- break;+ return true;default:return false;}++ // Only K-quant / IQ types reach here (they break out of the switch above)+ row_size_q8 = ggml_row_size(GGML_TYPE_Q8_K, ne00);return true;}