Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Upgrade to latest llama.cpp code
Justine Tunney0bdea60a92b1
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mkindex bd1dde8d..f09c18f5 100644--- a/llama.cpp/BUILD.mk+++ b/llama.cpp/BUILD.mk@@ -43,6 +43,11 @@ o/$(MODE)/llama.cpp/ggml-alloc.o \o/$(MODE)/llama.cpp/common.o: private \CCFLAGS += -Os+o/$(MODE)/llama.cpp/ggml-quants.o: private CXXFLAGS += -Os+o/$(MODE)/llama.cpp/ggml-quants-amd-avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge+o/$(MODE)/llama.cpp/ggml-quants-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2+o/$(MODE)/llama.cpp/ggml-quants-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f+$(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk.PHONY: o/$(MODE)/llama.cppdiff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex 0cca3351..d6eac66a 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -9,8 +9,8 @@ LICENSEORIGINhttps://github.com/ggerganov/llama.cpp/pull/4406/- 46e12c4692a37bdd31a0432fc5153d7d22bc7f72- 2024-04-25+ a8f9b076316e16aadd0791015b3bfd446fe1e904+ 2024-04-30LOCAL MODIFICATIONS@@ -21,9 +21,9 @@ LOCAL MODIFICATIONS- Add support to main() programs for Cosmo /zip/.args files- Introduce pledge() SECCOMP sandboxing to improve security- Call exit() rather than abort() when GGML_ASSERT() fails+ - Clamp bf16/f32 values before passing to K quantizers- Make GPU logger callback API safer and less generic- Write log to /dev/null when main.log fails to open- - Use _rand64() rather than time() as default seed- Make main and llava-cli print timings on ctrl-c- Avoid bind() conflicts on port 8080 w/ server- Use runtime dispatching for matmul quantsdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 0a777802..e57f083f 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -227,8 +227,54 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {return result;}+bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {+ const char * sep = strchr(data, '=');+ if (sep == nullptr || sep - data >= 128) {+ fprintf(stderr, "%s: malformed KV override '%s'\n", __func__, data);+ return false;+ }+ llama_model_kv_override kvo;+ std::strncpy(kvo.key, data, sep - data);+ kvo.key[sep - data] = 0;+ sep++;+ if (strncmp(sep, "int:", 4) == 0) {+ sep += 4;+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;+ kvo.val_i64 = std::atol(sep);+ } else if (strncmp(sep, "float:", 6) == 0) {+ sep += 6;+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;+ kvo.val_f64 = std::atof(sep);+ } else if (strncmp(sep, "bool:", 5) == 0) {+ sep += 5;+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;+ if (std::strcmp(sep, "true") == 0) {+ kvo.val_bool = true;+ } else if (std::strcmp(sep, "false") == 0) {+ kvo.val_bool = false;+ } else {+ fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data);+ return false;+ }+ } else if (strncmp(sep, "str:", 4) == 0) {+ sep += 4;+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;+ if (strlen(sep) > 127) {+ fprintf(stderr, "%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data);+ return false;+ }+ strncpy(kvo.val_str, sep, 127);+ kvo.val_str[127] = '\0';+ } else {+ fprintf(stderr, "%s: invalid type for KV override '%s'\n", __func__, data);+ return false;+ }+ overrides.emplace_back(std::move(kvo));+ return true;+}+bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param) {- llama_sampling_params& sparams = params.sparams;+ llama_sampling_params & sparams = params.sparams;if (arg == "--cli") {return true;@@ -878,7 +924,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_painvalid_param = true;return true;}- params.image = argv[i];+ params.image.emplace_back(argv[i]);return true;}if (arg == "-i" || arg == "--interactive") {@@ -933,6 +979,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paparams.cont_batching = true;return true;}+ if (arg == "-fa" || arg == "--flash-attn") {+ params.flash_attn = true;+ return true;+ }if (arg == "--color") {params.use_color = true;return true;@@ -1101,6 +1151,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paparams.n_print = std::stoi(argv[i]);return true;}+ if (arg == "--check-tensors") {+ params.check_tensors = true;+ return true;+ }if (arg == "--ppl-output-type") {if (++i >= argc) {invalid_param = true;@@ -1252,47 +1306,11 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_painvalid_param = true;return true;}- char* sep = strchr(argv[i], '=');- if (sep == nullptr || sep - argv[i] >= 128) {- fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);- invalid_param = true;- return true;- }- struct llama_model_kv_override kvo;- std::strncpy(kvo.key, argv[i], sep - argv[i]);- kvo.key[sep - argv[i]] = 0;- sep++;- if (strncmp(sep, "int:", 4) == 0) {- sep += 4;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;- kvo.int_value = std::atol(sep);- }- else if (strncmp(sep, "float:", 6) == 0) {- sep += 6;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;- kvo.float_value = std::atof(sep);- }- else if (strncmp(sep, "bool:", 5) == 0) {- sep += 5;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;- if (std::strcmp(sep, "true") == 0) {- kvo.bool_value = true;- }- else if (std::strcmp(sep, "false") == 0) {- kvo.bool_value = false;- }- else {- fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);- invalid_param = true;- return true;- }- }- else {+ if (!parse_kv_override(argv[i], params.kv_overrides)) {fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);invalid_param = true;return true;}- params.kv_overrides.push_back(kvo);return true;}#ifndef LOG_DISABLE_LOGS@@ -1322,6 +1340,29 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pareturn false;}+void gpt_params_handle_model_default(gpt_params & params) {+ if (!params.hf_repo.empty()) {+ // short-hand to avoid specifying --hf-file -> default it to --model+ if (params.hf_file.empty()) {+ if (params.model.empty()) {+ throw std::invalid_argument("error: --hf-repo requires either --hf-file or --model\n");+ }+ params.hf_file = params.model;+ } else if (params.model.empty()) {+ params.model = "models/" + string_split(params.hf_file, '/').back();+ }+ } else if (!params.model_url.empty()) {+ if (params.model.empty()) {+ auto f = string_split(params.model_url, '#').front();+ f = string_split(f, '?').front();+ f = string_split(f, '/').back();+ params.model = "models/" + f;+ }+ } else if (params.model.empty()) {+ params.model = DEFAULT_MODEL_PATH;+ }+}+bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {bool invalid_param = false;std::string arg;@@ -1350,10 +1391,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");}- // short-hand to avoid specifying --hf-file -> default it to --model- if (!params.hf_repo.empty() && params.hf_file.empty()) {- params.hf_file = params.model;- }+ gpt_params_handle_model_default(params);if (params.escape) {process_escapes(params.prompt);@@ -1494,8 +1532,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -ns N, --sequences N number of sequences to decode (default: %d)\n", params.n_sequences);printf(" -ps N, --p-split N speculative decoding split probability (default: %.1f)\n", (double)params.p_split);printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");+ printf(" -fa, --flash-attn enable Flash Attention (default: %s)\n", params.flash_attn ? "enabled" : "disabled");printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA. see examples/llava/README.md\n");- printf(" --image IMAGE_FILE path to an image file. use with multimodal models\n");+ printf(" --image IMAGE_FILE path to an image file. use with multimodal models. Specify multiple times for batching\n");if (llama_supports_mlock()) {printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");}@@ -1548,7 +1587,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --control-vector-layer-range START END\n");printf(" layer range to apply the control vector(s) to, start and end inclusive\n");printf(" -m FNAME, --model FNAME\n");- printf(" model path (default: %s)\n", params.model.c_str());+ printf(" model path (default: models/$filename with filename from --hf-file or --model-url if set, otherwise %s)\n", DEFAULT_MODEL_PATH);printf(" -md FNAME, --model-draft FNAME\n");printf(" draft model for speculative decoding (default: unused)\n");printf(" -mu MODEL_URL, --model-url MODEL_URL\n");@@ -1565,9 +1604,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" path to dynamic lookup cache to use for lookup decoding (updated by generation)\n");printf(" --override-kv KEY=TYPE:VALUE\n");printf(" advanced option to override model metadata by key. may be specified multiple times.\n");- printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");+ printf(" types: int, float, bool, str. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");printf(" -ptc N, --print-token-count N\n");printf(" print token count every N tokens (default: %d)\n", params.n_print);+ printf(" --check-tensors check model tensor data for invalid values\n");printf("\n");#ifndef LOG_DISABLE_LOGSlog_print_usage();@@ -1692,6 +1732,18 @@ std::vector<std::string> string_split(std::string input, char separator) {return parts;}+std::string string_strip(const std::string & str) {+ size_t start = 0;+ size_t end = str.size();+ while (start < end && std::isspace(str[start])) {+ start++;+ }+ while (end > start && std::isspace(str[end - 1])) {+ end--;+ }+ return str.substr(start, end - start);+}+std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names) {std::unordered_map<std::string, llama_sampler_type> sampler_canonical_name_map {{"top_k", llama_sampler_type::TOP_K},@@ -1788,6 +1840,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &mparams.tensor_split = params.tensor_split;mparams.use_mmap = params.use_mmap;mparams.use_mlock = params.use_mlock;+ mparams.check_tensors = params.check_tensors;if (params.kv_overrides.empty()) {mparams.kv_overrides = NULL;} else {@@ -1852,6 +1905,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_paramcparams.cb_eval = params.cb_eval;cparams.cb_eval_user_data = params.cb_eval_user_data;cparams.offload_kqv = !params.no_kv_offload;+ cparams.flash_attn = params.flash_attn;cparams.type_k = kv_cache_type_from_str(params.cache_type_k);cparams.type_v = kv_cache_type_from_str(params.cache_type_v);@@ -1882,59 +1936,75 @@ void llama_batch_add(#ifdef LLAMA_USE_CURL-static bool llama_download_file(CURL * curl, const char * url, const char * path) {+static bool starts_with(const std::string & str, const std::string & prefix) {+ // While we wait for C++20's std::string::starts_with...+ return str.rfind(prefix, 0) == 0;+}++static bool llama_download_file(const std::string & url, const std::string & path) {++ // Initialize libcurl+ std::unique_ptr<CURL, decltype(&curl_easy_cleanup)> curl(curl_easy_init(), &curl_easy_cleanup);+ if (!curl) {+ fprintf(stderr, "%s: error initializing libcurl\n", __func__);+ return false;+ }+bool force_download = false;// Set the URL, allow to follow http redirection- curl_easy_setopt(curl, CURLOPT_URL, url);- curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);+ curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());+ curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);#if defined(_WIN32)// CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of// operating system. Currently implemented under MS-Windows.- curl_easy_setopt(curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);+ curl_easy_setopt(curl.get(), CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);#endif// Check if the file already exists locallystruct stat model_file_info;- auto file_exists = (stat(path, &model_file_info) == 0);-- // If the file exists, check for ${path_model}.etag or ${path_model}.lastModified files- char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};- char etag_path[PATH_MAX] = {0};- snprintf(etag_path, sizeof(etag_path), "%s.etag", path);+ auto file_exists = (stat(path.c_str(), &model_file_info) == 0);- char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};- char last_modified_path[PATH_MAX] = {0};- snprintf(last_modified_path, sizeof(last_modified_path), "%s.lastModified", path);+ // If the file exists, check its JSON metadata companion file.+ std::string metadata_path = path + ".json";+ nlohmann::json metadata;+ std::string etag;+ std::string last_modified;if (file_exists) {- auto * f_etag = fopen(etag_path, "r");- if (f_etag) {- if (!fgets(etag, sizeof(etag), f_etag)) {- fprintf(stderr, "%s: unable to read file %s\n", __func__, etag_path);- } else {- fprintf(stderr, "%s: previous file found %s: %s\n", __func__, etag_path, etag);- }- fclose(f_etag);- }-- auto * f_last_modified = fopen(last_modified_path, "r");- if (f_last_modified) {- if (!fgets(last_modified, sizeof(last_modified), f_last_modified)) {- fprintf(stderr, "%s: unable to read file %s\n", __func__, last_modified_path);- } else {- fprintf(stderr, "%s: previous file found %s: %s\n", __func__, last_modified_path,- last_modified);+ // Try and read the JSON metadata file (note: stream autoclosed upon exiting this block).+ std::ifstream metadata_in(metadata_path);+ if (metadata_in.good()) {+ try {+ metadata_in >> metadata;+ fprintf(stderr, "%s: previous metadata file found %s: %s\n", __func__, metadata_path.c_str(), metadata.dump().c_str());+ if (metadata.contains("url") && metadata["url"].is_string()) {+ auto previous_url = metadata["url"].get<std::string>();+ if (previous_url != url) {+ fprintf(stderr, "%s: Model URL mismatch: %s != %s\n", __func__, url.c_str(), previous_url.c_str());+ return false;+ }+ }+ if (metadata.contains("etag") && metadata["etag"].is_string()) {+ etag = metadata["etag"];+ }+ if (metadata.contains("lastModified") && metadata["lastModified"].is_string()) {+ last_modified = metadata["lastModified"];+ }+ } catch (const nlohmann::json::exception & e) {+ fprintf(stderr, "%s: error reading metadata file %s: %s\n", __func__, metadata_path.c_str(), e.what());+ return false;}- fclose(f_last_modified);}+ } else {+ fprintf(stderr, "%s: no previous model file found %s\n", __func__, path.c_str());}// Send a HEAD request to retrieve the etag and last-modified headersstruct llama_load_model_from_url_headers {- char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};- char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};+ std::string etag;+ std::string last_modified;};llama_load_model_from_url_headers headers;{@@ -1942,38 +2012,37 @@ static bool llama_download_file(CURL * curl, const char * url, const char * pathauto header_callback = [](char * buffer, size_t /*size*/, size_t n_items, void * userdata) -> size_t {llama_load_model_from_url_headers *headers = (llama_load_model_from_url_headers *) userdata;- // Convert header field name to lowercase- for (size_t i = 0; i < n_items && buffer[i] != ':'; ++i) {- buffer[i] = tolower(buffer[i]);- }-- const char * etag_prefix = "etag: ";- if (strncmp(buffer, etag_prefix, strlen(etag_prefix)) == 0) {- strncpy(headers->etag, buffer + strlen(etag_prefix), n_items - strlen(etag_prefix) - 2); // Remove CRLF- }-- const char * last_modified_prefix = "last-modified: ";- if (strncmp(buffer, last_modified_prefix, strlen(last_modified_prefix)) == 0) {- strncpy(headers->last_modified, buffer + strlen(last_modified_prefix),- n_items - strlen(last_modified_prefix) - 2); // Remove CRLF+ static std::regex header_regex("([^:]+): (.*)\r\n");+ static std::regex etag_regex("ETag", std::regex_constants::icase);+ static std::regex last_modified_regex("Last-Modified", std::regex_constants::icase);++ std::string header(buffer, n_items);+ std::smatch match;+ if (std::regex_match(header, match, header_regex)) {+ const std::string & key = match[1];+ const std::string & value = match[2];+ if (std::regex_match(key, match, etag_regex)) {+ headers->etag = value;+ } else if (std::regex_match(key, match, last_modified_regex)) {+ headers->last_modified = value;+ }}return n_items;};- curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); // will trigger the HEAD verb- curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); // hide head request progress- curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, static_cast<CURLOPT_HEADERFUNCTION_PTR>(header_callback));- curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);+ curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 1L); // will trigger the HEAD verb+ curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1L); // hide head request progress+ curl_easy_setopt(curl.get(), CURLOPT_HEADERFUNCTION, static_cast<CURLOPT_HEADERFUNCTION_PTR>(header_callback));+ curl_easy_setopt(curl.get(), CURLOPT_HEADERDATA, &headers);- CURLcode res = curl_easy_perform(curl);+ CURLcode res = curl_easy_perform(curl.get());if (res != CURLE_OK) {- curl_easy_cleanup(curl);fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));return false;}long http_code = 0;- curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);+ curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &http_code);if (http_code != 200) {// HEAD not supported, we don't know if the file has changed// force trigger downloading@@ -1982,28 +2051,30 @@ static bool llama_download_file(CURL * curl, const char * url, const char * path}}- // If the ETag or the Last-Modified headers are different: trigger a new download- bool should_download = !file_exists- || force_download- || (strlen(headers.etag) > 0 && strcmp(etag, headers.etag) != 0)- || (strlen(headers.last_modified) > 0 && strcmp(last_modified, headers.last_modified) != 0);+ bool should_download = !file_exists || force_download;+ if (!should_download) {+ if (!etag.empty() && etag != headers.etag) {+ fprintf(stderr, "%s: ETag header is different (%s != %s): triggering a new download\n", __func__, etag.c_str(), headers.etag.c_str());+ should_download = true;+ } else if (!last_modified.empty() && last_modified != headers.last_modified) {+ fprintf(stderr, "%s: Last-Modified header is different (%s != %s): triggering a new download\n", __func__, last_modified.c_str(), headers.last_modified.c_str());+ should_download = true;+ }+ }if (should_download) {- char path_temporary[PATH_MAX] = {0};- snprintf(path_temporary, sizeof(path_temporary), "%s.downloadInProgress", path);+ std::string path_temporary = path + ".downloadInProgress";if (file_exists) {- fprintf(stderr, "%s: deleting previous downloaded file: %s\n", __func__, path);- if (remove(path) != 0) {- curl_easy_cleanup(curl);- fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path);+ fprintf(stderr, "%s: deleting previous downloaded file: %s\n", __func__, path.c_str());+ if (remove(path.c_str()) != 0) {+ fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path.c_str());return false;}}// Set the output file- auto * outfile = fopen(path_temporary, "wb");+ std::unique_ptr<FILE, decltype(&fclose)> outfile(fopen(path_temporary.c_str(), "wb"), fclose);if (!outfile) {- curl_easy_cleanup(curl);- fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path);+ fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path.c_str());return false;}@@ -2011,12 +2082,12 @@ static bool llama_download_file(CURL * curl, const char * url, const char * pathauto write_callback = [](void * data, size_t size, size_t nmemb, void * fd) -> size_t {return fwrite(data, size, nmemb, (FILE *)fd);};- curl_easy_setopt(curl, CURLOPT_NOBODY, 0L);- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));- curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile);+ curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 0L);+ curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));+ curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, outfile.get());// display download progress- curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);+ curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 0L);// helper function to hide password in URLauto llama_download_hide_password_in_url = [](const std::string & url) -> std::string {@@ -2035,51 +2106,34 @@ static bool llama_download_file(CURL * curl, const char * url, const char * path// start the downloadfprintf(stderr, "%s: downloading from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__,- llama_download_hide_password_in_url(url).c_str(), path, headers.etag, headers.last_modified);- auto res = curl_easy_perform(curl);+ llama_download_hide_password_in_url(url).c_str(), path.c_str(), headers.etag.c_str(), headers.last_modified.c_str());+ auto res = curl_easy_perform(curl.get());if (res != CURLE_OK) {- fclose(outfile);- curl_easy_cleanup(curl);fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));return false;}long http_code = 0;- curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);+ curl_easy_getinfo (curl.get(), CURLINFO_RESPONSE_CODE, &http_code);if (http_code < 200 || http_code >= 400) {- fclose(outfile);- curl_easy_cleanup(curl);fprintf(stderr, "%s: invalid http status code received: %ld\n", __func__, http_code);return false;}- // Clean up- fclose(outfile);+ // Causes file to be closed explicitly here before we rename it.+ outfile.reset();- // Write the new ETag to the .etag file- if (strlen(headers.etag) > 0) {- auto * etag_file = fopen(etag_path, "w");- if (etag_file) {- fputs(headers.etag, etag_file);- fclose(etag_file);- fprintf(stderr, "%s: file etag saved %s: %s\n", __func__, etag_path, headers.etag);- }- }-- // Write the new lastModified to the .etag file- if (strlen(headers.last_modified) > 0) {- auto * last_modified_file = fopen(last_modified_path, "w");- if (last_modified_file) {- fputs(headers.last_modified, last_modified_file);- fclose(last_modified_file);- fprintf(stderr, "%s: file last modified saved %s: %s\n", __func__, last_modified_path,- headers.last_modified);- }- }+ // Write the updated JSON metadata file.+ metadata.update({+ {"url", url},+ {"etag", headers.etag},+ {"lastModified", headers.last_modified}+ });+ std::ofstream(metadata_path) << metadata.dump(4);+ fprintf(stderr, "%s: file metadata saved: %s\n", __func__, metadata_path.c_str());- if (rename(path_temporary, path) != 0) {- curl_easy_cleanup(curl);- fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_temporary, path);+ if (rename(path_temporary.c_str(), path.c_str()) != 0) {+ fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str());return false;}}@@ -2097,15 +2151,7 @@ struct llama_model * llama_load_model_from_url(return NULL;}- // Initialize libcurl- auto * curl = curl_easy_init();-- if (!curl) {- fprintf(stderr, "%s: error initializing libcurl\n", __func__);- return NULL;- }-- if (!llama_download_file(curl, model_url, path_model)) {+ if (!llama_download_file(model_url, path_model)) {return NULL;}@@ -2119,7 +2165,6 @@ struct llama_model * llama_load_model_from_url(auto * ctx_gguf = gguf_init_from_file(path_model, gguf_params);if (!ctx_gguf) {fprintf(stderr, "\n%s: failed to load input GGUF from %s\n", __func__, path_model);- curl_easy_cleanup(curl);return NULL;}@@ -2131,8 +2176,6 @@ struct llama_model * llama_load_model_from_url(gguf_free(ctx_gguf);}- curl_easy_cleanup(curl);-if (n_split > 1) {char split_prefix[PATH_MAX] = {0};char split_url_prefix[LLAMA_CURL_MAX_URL_LENGTH] = {0};@@ -2163,11 +2206,7 @@ struct llama_model * llama_load_model_from_url(char split_url[LLAMA_CURL_MAX_URL_LENGTH] = {0};llama_split_path(split_url, sizeof(split_url), split_url_prefix, download_idx, n_split);- auto * curl = curl_easy_init();- bool res = llama_download_file(curl, split_url, split_path);- curl_easy_cleanup(curl);-- return res;+ return llama_download_file(split_url, split_path);}, idx));}@@ -2654,7 +2693,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau);fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta);fprintf(stream, "mlock: %s # default: false\n", params.use_mlock ? "true" : "false");- fprintf(stream, "model: %s # default: models/7B/ggml-model.bin\n", params.model.c_str());+ fprintf(stream, "model: %s # default: %s\n", params.model.c_str(), DEFAULT_MODEL_PATH);fprintf(stream, "model_draft: %s # default:\n", params.model_draft.c_str());fprintf(stream, "multiline_input: %s # default: false\n", params.multiline_input ? "true" : "false");fprintf(stream, "n_gpu_layers: %d # default: -1\n", params.n_gpu_layers);@@ -2689,6 +2728,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "seed: %u # default: -1 (random seed)\n", params.seed);fprintf(stream, "simple_io: %s # default: false\n", params.simple_io ? "true" : "false");fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false");+ fprintf(stream, "flash_attn: %s # default: false\n", params.flash_attn ? "true" : "false");fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp);const std::vector<float> tensor_split_vector(params.tensor_split, params.tensor_split + llama_max_devices());diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 75fcc901..2cbd376c 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -36,6 +36,8 @@tinylog(__func__, ": llamafile version " LLAMAFILE_VERSION_STRING "\n", NULL); \} while(0)+#define DEFAULT_MODEL_PATH "models/7B/ggml-model-f16.gguf"+// build infoextern int LLAMA_BUILD_NUMBER;extern char const *LLAMA_COMMIT;@@ -97,7 +99,7 @@ struct gpt_params {// // sampling parametersstruct llama_sampling_params sparams;- std::string model = "models/7B/ggml-model-f16.gguf"; // model path+ std::string model = ""; // model pathstd::string model_draft = ""; // draft model for speculative decodingstd::string model_alias = "unknown"; // model aliasstd::string model_url = ""; // model url to download@@ -138,7 +140,7 @@ struct gpt_params {bool multiple_choice = false; // compute TruthfulQA score over random tasks from datafile supplied in promptsize_t multiple_choice_tasks = 0; // number of tasks to use when computing the TruthfulQA score. If 0, all tasks will be computed- bool kl_divergence = false; // compute KL-divergence+ bool kl_divergence = false; // compute KL divergencebool random_prompt = false; // do not randomize prompt if none providedbool use_color = false; // use color to distinguish generations and inputs@@ -153,6 +155,7 @@ struct gpt_params {bool multiline_input = false; // reverse the usage of `\`bool simple_io = false; // improves compatibility with subprocesses and limited consolesbool cont_batching = true; // insert new sequences for decoding on-the-fly+ bool flash_attn = false; // flash attentionbool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefixbool ignore_eos = false; // ignore generated EOS tokens@@ -166,15 +169,20 @@ struct gpt_params {bool dump_kv_cache = false; // dump the KV cache contents for debugging purposesbool no_kv_offload = false; // disable KV offloadingbool warmup = true; // warmup run+ bool check_tensors = false; // validate tensor datastd::string cache_type_k = "f16"; // KV cache data type for the Kstd::string cache_type_v = "f16"; // KV cache data type for the V// multimodal models (see examples/llava)- std::string mmproj = ""; // path to multimodal projector- std::string image = ""; // path to an image file+ std::string mmproj = ""; // path to multimodal projector+ std::vector<std::string> image; // path to image file(s)};+void gpt_params_handle_model_default(gpt_params & params);++bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides);+bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params);bool gpt_params_parse(int argc, char ** argv, gpt_params & params);@@ -198,6 +206,7 @@ bool validate_file_name(const std::string & filename);std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);std::vector<llama_sampler_type> sampler_types_from_chars(const std::string & names_string);std::vector<std::string> string_split(std::string input, char separator);+std::string string_strip(const std::string & str);std::string sampler_type_to_name_string(llama_sampler_type sampler_type);//diff --git a/llama.cpp/ggml-backend-impl.h b/llama.cpp/ggml-backend-impl.hindex 4871987f..f02379c7 100644--- a/llama.cpp/ggml-backend-impl.h+++ b/llama.cpp/ggml-backend-impl.h@@ -180,6 +180,7 @@ extern "C" {bool (*GGML_CALL ggml_backend_buffer_is_host)(ggml_backend_buffer_t);bool (*GGML_CALL ggml_guid_matches)(ggml_guid_t, ggml_guid_t);bool (*GGML_CALL ggml_is_empty)(const struct ggml_tensor *);+ bool (*GGML_CALL ggml_are_same_shape)(const struct ggml_tensor *, const struct ggml_tensor *);};#ifdef __cplusplusdiff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex 749bddc2..c32aa53f 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -1790,12 +1790,14 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {void ggml_backend_sched_reset(ggml_backend_sched_t sched) {// reset state for the next run- size_t hash_size = sched->hash_set.size;- memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); // NOLINT- memset(sched->tensor_backend_id, -1, sizeof(sched->tensor_backend_id[0]) * hash_size);- memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size);+ if (!sched->is_reset) {+ size_t hash_size = sched->hash_set.size;+ memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); // NOLINT+ memset(sched->tensor_backend_id, -1, sizeof(sched->tensor_backend_id[0]) * hash_size);+ memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size);- sched->is_reset = true;+ sched->is_reset = true;+ }sched->is_alloc = false;}@@ -2161,6 +2163,7 @@ static const struct ggml_backend_api kGgmlBackendApi = {ggml_backend_buffer_is_host,ggml_guid_matches,ggml_is_empty,+ ggml_are_same_shape,};const struct ggml_backend_api *ggml_backend_api(void) {diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 09c73c6a..83b1dc80 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -1,9 +1,8 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-static void ggml_cuda_print(int, const char *, ...);-#include <algorithm>+#include <stdarg.h>#include <array>#include <atomic>#include <cassert>@@ -17,9 +16,10 @@ static void ggml_cuda_print(int, const char *, ...);#include <map>#include <memory>#include <mutex>+#include <stdint.h>+#include <stdio.h>#include <string>#include <vector>-#include <stdarg.h>#include "ggml.h"#include "ggml-cuda.h"@@ -298,6 +298,7 @@ GGML_CALL bool ggml_cuda_link(const struct ggml_backend_api *backend_api) {#define CC_PASCAL 600#define MIN_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products#define CC_VOLTA 700+#define CC_AMPERE 800#define CC_OFFSET_AMD 1000000#define CC_RDNA1 (CC_OFFSET_AMD + 1010)#define CC_RDNA2 (CC_OFFSET_AMD + 1030)@@ -427,7 +428,6 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {return a;}-#ifdef GGML_CUDA_F16static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL#pragma unroll@@ -440,7 +440,6 @@ static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {NO_DEVICE_CODE;#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL}-#endif // GGML_CUDA_F16static __device__ __forceinline__ float warp_reduce_max(float x) {#pragma unroll@@ -450,19 +449,26 @@ static __device__ __forceinline__ float warp_reduce_max(float x) {return x;}-//static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {-//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX-//#pragma unroll-// for (int mask = 16; mask > 0; mask >>= 1) {-// x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));-// }-// return x;-//#else-// GGML_UNUSED(x);-// NO_DEVICE_CODE;-//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX-//}+static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));+ }+ return x;+#else+ GGML_UNUSED(x);+ NO_DEVICE_CODE;+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX+}+#if CUDART_VERSION < 12000+static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {+ const uint32_t mask_low = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b)));+ const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b)));+ return mask_low | mask_high;+}+#endif // CUDART_VERSION < 12000#if defined(GGML_USE_HIPBLAS)#define __CUDA_ARCH__ 1300@@ -547,6 +553,15 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {}#endif // defined(GGML_USE_HIPBLAS)+#define FP16_AVAILABLE defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) ? \+ defined(RDNA1) || defined(RDNA2) || defined(RDNA3) : __CUDA_ARCH__ >= CC_PASCAL++#define FP16_MMA_AVAILABLE !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA++#if FP16_MMA_AVAILABLE+#include <mma.h>+#endif+// TODO: move to ggml-common.hstatic const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};@@ -560,6 +575,7 @@ struct ggml_cuda_device_info {struct cuda_device_info {int cc; // compute capability+ int nsm; // number of streaming multiprocessorssize_t smpb; // max. shared memory per blockbool vmm; // virtual memory supportsize_t vmm_granularity; // granularity of virtual memory@@ -772,6 +788,8 @@ void ggml_cuda_op_dequantize_mul_mat_vec(const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,const int64_t src1_padded_row_size, cudaStream_t stream);+void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+#define CUDA_GET_ROWS_BLOCK_SIZE 256void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);@@ -2849,16 +2867,16 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {- const int64_t i = 2*(blockDim.x*blockIdx.x + threadIdx.x);+ const int64_t i = (int64_t)2*(blockDim.x*blockIdx.x + threadIdx.x);if (i >= k) {return;}const int64_t ib = i/qk; // block index- const int iqs = (i%qk)/qr; // quant index- const int iybs = i - i%qk; // y block start index- const int y_offset = qr == 1 ? 1 : qk/2;+ const int64_t iqs = (i%qk)/qr; // quant index+ const int64_t iybs = i - i%qk; // y block start index+ const int64_t y_offset = qr == 1 ? 1 : qk/2;// dequantizedfloat2 v;@@ -2873,7 +2891,7 @@ static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, h#if __CUDA_ARCH__ >= CC_PASCALconstexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;- const int i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;+ const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;const int * x0 = ((int *) vx) + blockIdx.x * nint;half2 * y2 = (half2 *) (y + i0);@@ -2917,9 +2935,9 @@ static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_tconst int64_t i = blockIdx.x;// assume 32 threads- const int tid = threadIdx.x;- const int il = tid/8;- const int ir = tid%8;+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;const int64_t ib = 8*i + ir;if (ib >= nb32) {return;@@ -2945,9 +2963,9 @@ static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_tconst int64_t i = blockIdx.x;// assume 32 threads- const int tid = threadIdx.x;- const int il = tid/8;- const int ir = tid%8;+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;const int64_t ib = 8*i + ir;if (ib >= nb32) {return;@@ -2971,14 +2989,14 @@ static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_ttemplate<typename dst_t>static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_q2_K * x = (const block_q2_K *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int n = tid/32;- const int l = tid - 32*n;- const int is = 8*n + l/16;+ const int64_t n = tid/32;+ const int64_t l = tid - 32*n;+ const int64_t is = 8*n + l/16;const uint8_t q = x[i].qs[32*n + l];dst_t * y = yy + i*QK_K + 128*n;@@ -2990,8 +3008,8 @@ static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_ty[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);#else- const int is = tid/16; // 0 or 1- const int il = tid%16; // 0...15+ const int64_t is = tid/16; // 0 or 1+ const int64_t il = tid%16; // 0...15const uint8_t q = x[i].qs[il] >> (2*is);dst_t * y = yy + i*QK_K + 16*is + il;float dall = __low2half(x[i].dm);@@ -3005,19 +3023,19 @@ static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_ttemplate<typename dst_t>static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_q3_K * x = (const block_q3_K *) vx;#if QK_K == 256- const int r = threadIdx.x/4;- const int tid = r/2;- const int is0 = r%2;- const int l0 = 16*is0 + 4*(threadIdx.x%4);- const int n = tid / 4;- const int j = tid - 4*n;+ const int64_t r = threadIdx.x/4;+ const int64_t tid = r/2;+ const int64_t is0 = r%2;+ const int64_t l0 = 16*is0 + 4*(threadIdx.x%4);+ const int64_t n = tid / 4;+ const int64_t j = tid - 4*n;uint8_t m = 1 << (4*n + j);- int is = 8*n + 2*j + is0;+ int64_t is = 8*n + 2*j + is0;int shift = 2*j;int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :@@ -3033,11 +3051,11 @@ static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_tfor (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));#else- const int tid = threadIdx.x;- const int is = tid/16; // 0 or 1- const int il = tid%16; // 0...15- const int im = il/8; // 0...1- const int in = il%8; // 0...7+ const int64_t tid = threadIdx.x;+ const int64_t is = tid/16; // 0 or 1+ const int64_t il = tid%16; // 0...15+ const int64_t im = il/8; // 0...1+ const int64_t in = il%8; // 0...7dst_t * y = yy + i*QK_K + 16*is + il;@@ -3071,15 +3089,15 @@ template<typename dst_t>static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {const block_q4_K * x = (const block_q4_K *) vx;- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;#if QK_K == 256// assume 32 threads- const int tid = threadIdx.x;- const int il = tid/8;- const int ir = tid%8;- const int is = 2*il;- const int n = 4;+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t is = 2*il;+ const int64_t n = 4;dst_t * y = yy + i*QK_K + 64*il + n*ir;@@ -3098,7 +3116,7 @@ static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_ty[l +32] = d2 * (q[l] >> 4) - m2;}#else- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;const uint8_t * q = x[i].qs;dst_t * y = yy + i*QK_K;const float d = (float)x[i].dm[0];@@ -3112,14 +3130,14 @@ template<typename dst_t>static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {const block_q5_K * x = (const block_q5_K *) vx;- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;#if QK_K == 256// assume 64 threads - this is very slightly better than the one below- const int tid = threadIdx.x;- const int il = tid/16; // il is in 0...3- const int ir = tid%16; // ir is in 0...15- const int is = 2*il; // is is in 0...6+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/16; // il is in 0...3+ const int64_t ir = tid%16; // ir is in 0...15+ const int64_t is = 2*il; // is is in 0...6dst_t * y = yy + i*QK_K + 64*il + 2*ir;@@ -3142,11 +3160,11 @@ static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_ty[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;#else- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;const uint8_t q = x[i].qs[tid];- const int im = tid/8; // 0...3- const int in = tid%8; // 0...7- const int is = tid/16; // 0 or 1+ const int64_t im = tid/8; // 0...3+ const int64_t in = tid%8; // 0...7+ const int64_t is = tid/16; // 0 or 1const uint8_t h = x[i].qh[in] >> im;const float d = x[i].d;dst_t * y = yy + i*QK_K + tid;@@ -3203,13 +3221,13 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_ttemplate<typename dst_t>static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq2_xxs * x = (const block_iq2_xxs *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 8*il;const uint16_t * q2 = x[i].qs + 4*ib;const uint8_t * aux8 = (const uint8_t *)q2;@@ -3227,13 +3245,13 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dstemplate<typename dst_t>static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq2_xs * x = (const block_iq2_xs *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 8*il;const uint16_t * q2 = x[i].qs + 4*ib;const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));@@ -3249,13 +3267,13 @@ static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dsttemplate<typename dst_t>static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq2_s * x = (const block_iq2_s *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 8*il;const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;@@ -3270,13 +3288,13 @@ static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_template<typename dst_t>static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq3_xxs * x = (const block_iq3_xxs *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 8*il;const uint8_t * q3 = x[i].qs + 8*ib;const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;@@ -3298,13 +3316,13 @@ static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dstemplate<typename dst_t>static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq3_s * x = (const block_iq3_s *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 8*il;const uint8_t * qs = x[i].qs + 8*ib;const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));@@ -3324,13 +3342,13 @@ static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_template<typename dst_t>static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq1_s * x = (const block_iq1_s *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 8*il;const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);@@ -3350,18 +3368,18 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_template<typename dst_t>static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq1_m * x = (const block_iq1_m *) vx;- const int tid = threadIdx.x;+ const int64_t tid = threadIdx.x;#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 8*il;const uint16_t * sc = (const uint16_t *)x[i].scales;iq1m_scale_t scale;scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);- const int ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);+ const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;@@ -3381,12 +3399,12 @@ static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_template<typename dst_t>static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);- const int tid = threadIdx.x;- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 4*il;const uint8_t * q4 = x[ib].qs + 4*il;const float d = (float)x[ib].d;@@ -3400,12 +3418,12 @@ static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst#if QK_K != 64template<typename dst_t>static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int i = blockIdx.x;+ const int64_t i = blockIdx.x;const block_iq4_xs * x = (const block_iq4_xs *)vx;- const int tid = threadIdx.x;- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7dst_t * y = yy + i*QK_K + 32*ib + 4*il;const uint8_t * q4 = x[i].qs + 16*ib + 4*il;const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);@@ -4978,6 +4996,942 @@ void ggml_cuda_op_dequantize_mul_mat_vec(GGML_UNUSED(src1_padded_row_size);}+#define FATTN_KQ_STRIDE 256+#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction.+#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs.++template<int D, int parallel_blocks> // D == head size+__launch_bounds__(((D + WARP_SIZE - 1) / WARP_SIZE)*WARP_SIZE, 1)+static __global__ void flash_attn_vec_ext_f16(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3) {+#if FP16_AVAILABLE+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.++ const int ic = blockIdx.x / parallel_blocks; // Index of the Q/QKV column to work on.+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.++ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic);+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape+ const half * maskh = (const half *) mask + ne11*ic;++ const int stride_KV = nb11 / sizeof(half);+ const int stride_KV2 = nb11 / sizeof(half2);++ constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE;+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;+ __builtin_assume(tid < nwarps*WARP_SIZE);++ __shared__ half KQ[nwarps*WARP_SIZE];+ KQ[tid] = -INFINITY;+ half2 * KQ2 = (half2 *) KQ;++ half kqmax = -HALF_MAX_HALF;+ half kqsum = 0.0f;++ __shared__ half kqmax_shared[WARP_SIZE];+ __shared__ half kqsum_shared[WARP_SIZE];+ if (threadIdx.y == 0) {+ kqmax_shared[threadIdx.x] = -HALF_MAX_HALF;+ kqsum_shared[threadIdx.x] = 0.0f;+ }+ __syncthreads();++ // Convert Q to half2 and store in registers:+ half2 Q_h2[(D/2 + WARP_SIZE - 1) / WARP_SIZE];+#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {+ break;+ }++ Q_h2[i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(Q_f2[i].x, Q_f2[i].y);+ }++ half2 VKQ = make_half2(0.0f, 0.0f); // Each thread calculates a single VKQ value.++ const int k_start = parallel_blocks == 1 ? 0 : ip*D;+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) {+ // Calculate KQ tile and keep track of new maximum KQ values:+ half kqmax_new = kqmax;+#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) {+ const int i_KQ = i_KQ_0 + threadIdx.y;++ if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) {+ break;+ }++ half2 sum2 = make_half2(0.0f, 0.0f);+#pragma unroll+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {+ const int k_KQ = k_KQ_0 + threadIdx.x;+ if (k_KQ_0 + WARP_SIZE > D/2 && k_KQ >= D/2) {+ break;+ }++ const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ];+ sum2 += K_ik * Q_h2[k_KQ_0/WARP_SIZE];+ }++ sum2 = warp_reduce_sum(sum2);+ half sum = __low2half(sum2) + __high2half(sum2);+ sum += mask ? maskh[k_VKQ_0 + i_KQ] : __float2half(0.0f);+ kqmax_new = __hmax(kqmax_new, sum);+ if (threadIdx.x == 0) {+ KQ[i_KQ] = sum;+ }+ }++ kqmax_new = warp_reduce_max(kqmax_new);+ if (threadIdx.x == 0) {+ kqmax_shared[threadIdx.y] = kqmax_new;+ }+ __syncthreads();+ kqmax_new = kqmax_shared[threadIdx.x];+ kqmax_new = warp_reduce_max(kqmax_new);++ const half KQ_max_scale = hexp(kqmax - kqmax_new);+ kqmax = kqmax_new;++ const half val = hexp(KQ[tid] - kqmax);+ kqsum = kqsum*KQ_max_scale + val;+ KQ[tid] = val;++ VKQ *= __half2half2(KQ_max_scale);++ __syncthreads();++ if (tid < D) {+#pragma unroll+ for (int k0 = 0; k0 < D; k0 += 2) {+ if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k0 >= ne11) {+ break;+ }++ half2 V_k;+ reinterpret_cast<half&>(V_k.x) = V_h[(k_VKQ_0 + k0 + 0)*stride_KV + tid];+ reinterpret_cast<half&>(V_k.y) = V_h[(k_VKQ_0 + k0 + 1)*stride_KV + tid];+ VKQ += V_k*KQ2[k0/2];+ }+ }++ __syncthreads();+ }++ if (tid >= D) {+ kqsum = 0.0f;+ }++ kqsum = warp_reduce_sum(kqsum);+ if (threadIdx.x == 0) {+ kqsum_shared[threadIdx.y] = kqsum;+ }+ __syncthreads();+ kqsum = kqsum_shared[threadIdx.x];+ kqsum = warp_reduce_sum(kqsum);++ if (tid >= D) {+ return;+ }++ half dst_val = (__low2half(VKQ) + __high2half(VKQ));+ if (parallel_blocks == 1) {+ dst_val /= kqsum;+ }+ dst[D*gridDim.y*blockIdx.x + D*blockIdx.y + tid] = dst_val;++ if (parallel_blocks == 1 || tid != 0) {+ return;+ }+ dst_meta[ic*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax, kqsum);+#else+ NO_DEVICE_CODE;+#endif // FP16_AVAILABLE+}++// D == head size, VKQ_stride == num VKQ rows calculated in parallel:+template<int D, int ncols, int nwarps, int VKQ_stride, int parallel_blocks, typename KQ_acc_t>+__launch_bounds__(nwarps*WARP_SIZE, 1)+static __global__ void flash_attn_ext_f16(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3) {+#if FP16_MMA_AVAILABLE+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.++ const int ic0 = ncols*(blockIdx.x / parallel_blocks); // Index of the first Q/QKV column to work on.+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.++ static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE.");+ static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16.");+ constexpr int frag_m = ncols == 8 ? 32 : 16;+ constexpr int frag_n = ncols == 8 ? 8 : 16;+ static_assert(D % frag_m == 0, "If ncols == 8 then D % frag_m must be 0.");+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::row_major> frag_a_K;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_a_V;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_b;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, KQ_acc_t> frag_c_KQ;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, half> frag_c_VKQ;++ constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel.+ constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy.+ static_assert(VKQ_ratio <= nwarps, "VKQ_ratio must be <= nwarps.");++ // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts:+ constexpr int D_padded = D + 8;+ constexpr int kqs_padded = FATTN_KQ_STRIDE + 8;+ constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half);++ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.+ const float * Q_f = (const float *) (Q + nb02* blockIdx.y + nb01*ic0);+ const half * K_h = (const half *) (K + nb12*(blockIdx.y / gqa_ratio));+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape+ const half * maskh = (const half *) mask + (nb31/sizeof(half))* ic0;+ const half2 * mask2 = (const half2 *) mask + (nb31/sizeof(half))*(ic0/2);++ const int stride_Q = nb01 / sizeof(float);+ const int stride_KV = nb11 / sizeof(half);++ frag_b Q_b[D/16][ncols/frag_n];++ // A single buffer for temporarily holding tiles of KQ and VKQ parts:+ constexpr int mem_KQ = ncols*kqs_padded*kqar;+ constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded;+ __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts];+ float * KQ_f = (float *) KQ;+ half2 * KQ2 = (half2 *) KQ;++ float KQ_rowsum_f[ncols/nwarps] = {0.0f};+ float KQ_max_f[ncols/nwarps];+ float KQ_max_scale_f[ncols/nwarps] = {0.0f};++#pragma unroll+ for (int j = 0; j < ncols/nwarps; ++j) {+ KQ_max_f[j] = -FLT_MAX/2.0f;+ }++ half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}};+ half2 KQ_max_h2[ncols/nwarps];+ half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}};++#pragma unroll+ for (int j = 0; j < ncols/nwarps; ++j) {+ KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF);+ }++ __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice.+ half2 * VKQ2 = (half2 *) VKQ;+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;+#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {+ break;+ }+ VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f);+ }+ }++ // Convert Q to half and apply scale, temporarily store in KQ:+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;+#pragma unroll+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D && i >= D) {+ break;+ }+ KQ[j*D_padded + i] = ic0 + j < ne01 ? Q_f[j*stride_Q + i] * scale : 0.0f;+ }+ }++ __syncthreads();++ // Load Q into tensor core fragments/registers since it will be used frequently:+#pragma unroll+ for (int i0 = 0; i0 < D; i0 += 16) {+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+ nvcuda::wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ + j0*D_padded + i0, D_padded);+ }+ }++ __syncthreads();++ // Iterate over ne11 == previous tokens:+ for (int k_VKQ_0 = ip*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE) {+ // Calculate tile of KQ:+#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) {+ frag_c_KQ KQ_c[ncols/frag_n];+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::fill_fragment(KQ_c[j], 0.0f);+ }+#pragma unroll+ for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) {+ frag_a_K K_a;+ nvcuda::wmma::load_matrix_sync(K_a, K_h + (k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV);+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]);+ }+ }+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+ nvcuda::wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, nvcuda::wmma::mem_col_major);+ }+ }++ __syncthreads();++ // Calculate softmax for each KQ column using the current max. value.+ // The divisor is stored in KQ_rowsum and will be applied at the end.+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ if (std::is_same<KQ_acc_t, float>::value) {+ float KQ_f_tmp[FATTN_KQ_STRIDE / WARP_SIZE];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ_f_tmp[k0/WARP_SIZE] = KQ_f[j*kqs_padded + k];+ }++ float KQ_max_new = KQ_max_f[j0/nwarps];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ_f_tmp[k0/WARP_SIZE] += mask ? __half2float(maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f;+ KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/WARP_SIZE]);+ }+ KQ_max_new = warp_reduce_max(KQ_max_new);++ const float diff = KQ_max_f[j0/nwarps] - KQ_max_new;+ KQ_max_scale_f[j0/nwarps] = expf(diff);+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {+ KQ_max_scale_f[j0/nwarps] = 0.0f;+ }+ KQ_max_f[j0/nwarps] = KQ_max_new;++ float KQ_rowsum_add = 0.0f;+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ const float diff = KQ_f_tmp[k0/WARP_SIZE] - KQ_max_f[j0/nwarps];+ KQ_f_tmp[k0/WARP_SIZE] = expf(diff);+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {+ KQ_f_tmp[k0/WARP_SIZE] = 0.0f;+ }+ KQ_rowsum_add += KQ_f_tmp[k0/WARP_SIZE];+ KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/WARP_SIZE];+ }+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);++ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:+ KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add;+ } else {+ half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*WARP_SIZE)];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ2_tmp[k0/WARP_SIZE] = KQ2[j*(kqs_padded/2) + k];+ }++ half2 KQ_max_new = KQ_max_h2[j0/nwarps];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ2_tmp[k0/WARP_SIZE] += mask ? mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f);+ KQ_max_new = __hmax2(KQ_max_new, KQ2_tmp[k0/WARP_SIZE]);+ }+ KQ_max_new = __half2half2(warp_reduce_max(__hmax(__low2half(KQ_max_new), __high2half(KQ_max_new))));+ const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new;+ KQ_max_scale_h2[j0/nwarps] = h2exp(diff);+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));+ *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask;+ KQ_max_h2[j0/nwarps] = KQ_max_new;++ half2 KQ_rowsum_add = make_half2(0.0f, 0.0f);+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ const half2 diff = KQ2_tmp[k0/WARP_SIZE] - KQ_max_h2[j0/nwarps];+ KQ2_tmp[k0/WARP_SIZE] = h2exp(diff);+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));+ *((uint32_t *) &KQ2_tmp[k0/WARP_SIZE]) &= ftz_mask;+ KQ_rowsum_add += KQ2_tmp[k0/WARP_SIZE];+ KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/WARP_SIZE];+ }+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);++ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:+ KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add;+ }+ }++ __syncthreads();++ frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n];+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;+ nvcuda::wmma::load_matrix_sync(+ KQ_b[k0/(VKQ_ratio*16)][j0/frag_n],+ KQ + j0*(kqar*kqs_padded) + k,+ kqar*kqs_padded);+ }+ }++ frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n];+#pragma unroll+ for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) {+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], 0.0f);+ }++#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;++ frag_a_V v_a;+ nvcuda::wmma::load_matrix_sync(v_a, V_h + (k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV);+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]);+ }+ }+ }++ __syncthreads();++ const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded);+#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) {+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+ nvcuda::wmma::store_matrix_sync(+ KQ + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio),+ VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n],+ D_padded, nvcuda::wmma::mem_col_major);+ }+ }++ __syncthreads();++#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ half2 VKQ_scale;+ if (std::is_same<KQ_acc_t, float>::value) {+ VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]);+ } else {+ VKQ_scale = KQ_max_scale_h2[j0/nwarps];+ }++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {+ break;+ }++ half2 VKQ_add = make_half2(0.0f, 0.0f);+#pragma unroll+ for (int l = 0; l < VKQ_ratio; ++l) {+ VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i];+ }+ VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add;+ }+ }++ __syncthreads();+ }++#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j_VKQ = j0 + threadIdx.y;+ if (ic0 + j_VKQ >= ne01) {+ return;+ }+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;++ float KQ_rowsum_j;+ if (std::is_same<KQ_acc_t, float>::value) {+ KQ_rowsum_j = KQ_rowsum_f[j0/nwarps];+ } else {+ KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]);+ }++#pragma unroll+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D && i >= D) {+ break;+ }+ float dst_val = VKQ[j_VKQ*D_padded + i];+ if (parallel_blocks == 1) {+ dst_val /= KQ_rowsum_j;+ }+ dst[j_dst*gridDim.y*D + blockIdx.y*D + i] = dst_val;+ }++ if (parallel_blocks == 1 || threadIdx.x != 0) {+ continue;+ }++ float2 dst_meta_val;+ if (std::is_same<KQ_acc_t, float>::value) {+ dst_meta_val.x = KQ_max_f[j0/nwarps];+ } else {+ dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]);+ }+ dst_meta_val.y = KQ_rowsum_j;+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = dst_meta_val;+ }+#else+ NO_DEVICE_CODE;+#endif // FP16_MMA_AVAILABLE+}++template<int D, int parallel_blocks> // D == head size+__launch_bounds__(D, 1)+static __global__ void flash_attn_combine_results(+ const float * __restrict__ VKQ_parts,+ const float2 * __restrict__ VKQ_meta,+ float * __restrict__ dst) {+#if FP16_AVAILABLE+ VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x;+ VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x;+ dst += D * gridDim.y*blockIdx.x;++ const int tid = threadIdx.x;+ __builtin_assume(tid < D);++ __shared__ float2 meta[parallel_blocks];+ if (tid < 2*parallel_blocks) {+ ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid];+ }++ __syncthreads();++ float kqmax = meta[0].x;+#pragma unroll+ for (int l = 1; l < parallel_blocks; ++l) {+ kqmax = max(kqmax, meta[l].x);+ }++ float VKQ_numerator = 0.0f;+ float VKQ_denominator = 0.0f;+#pragma unroll+ for (int l = 0; l < parallel_blocks; ++l) {+ const float diff = meta[l].x - kqmax;+ const float KQ_max_scale = expf(diff);+ const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD);+ *((uint32_t *) &KQ_max_scale) &= ftz_mask;++ VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid];+ VKQ_denominator += KQ_max_scale * meta[l].y;+ }++ dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator;+#else+ NO_DEVICE_CODE;+#endif // FP16_AVAILABLE+}++constexpr int get_max_power_of_2(int x) {+ return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1;+}++static_assert(get_max_power_of_2(1) == 1, "Test failed.");+static_assert(get_max_power_of_2(2) == 2, "Test failed.");+static_assert(get_max_power_of_2(4) == 4, "Test failed.");+static_assert(get_max_power_of_2(6) == 2, "Test failed.");++// Number of VKQ rows calculated in parallel:+constexpr int get_VKQ_stride(int D, int nwarps, int frag_m) {+ return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m;+}++static_assert(get_VKQ_stride(128, 1, 32) == 32, "Test failed.");+static_assert(get_VKQ_stride(128, 2, 32) == 64, "Test failed.");+static_assert(get_VKQ_stride(128, 4, 32) == 128, "Test failed.");+static_assert(get_VKQ_stride( 64, 1, 32) == 32, "Test failed.");+static_assert(get_VKQ_stride( 64, 2, 32) == 64, "Test failed.");+static_assert(get_VKQ_stride( 64, 4, 32) == 64, "Test failed.");+static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed.");+static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed.");+static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed.");++template <int D, int parallel_blocks> void launch_fattn_vec_f16(+ const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,+ ggml_cuda_pool & pool, cudaStream_t main_stream+) {+ ggml_cuda_pool_alloc<float> dst_tmp(pool);+ ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);++ if (parallel_blocks > 1) {+ dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));+ dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));+ }++ constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE;+ const dim3 block_dim(WARP_SIZE, nwarps, 1);+ const dim3 blocks_num(parallel_blocks*Q->ne[1], Q->ne[2], Q->ne[3]);+ const int shmem = 0;++ float scale;+ memcpy(&scale, KQV->op_params, sizeof(float));++ flash_attn_vec_ext_f16<D, parallel_blocks>+ <<<blocks_num, block_dim, shmem, main_stream>>> (+ (const char *) Q->data,+ (const char *) K->data,+ (const char *) V->data,+ mask ? ((const char *) mask->data) : nullptr,+ parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,+ scale,+ Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],+ K->ne[0], K->ne[1], K->ne[2], K->ne[3],+ mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,+ Q->nb[1], Q->nb[2], Q->nb[3],+ K->nb[1], K->nb[2], K->nb[3],+ KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]+ );+ CUDA_CHECK(cudaGetLastError());++ if (parallel_blocks == 1) {+ return;+ }++ const dim3 block_dim_combine(D, 1, 1);+ const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);+ const int shmem_combine = 0;++ flash_attn_combine_results<D, parallel_blocks>+ <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>+ (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);+ CUDA_CHECK(cudaGetLastError());+}++template <int D, int cols_per_block, int nwarps, int parallel_blocks, typename KQ_acc_t> void launch_fattn_f16_impl(+ const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,+ ggml_cuda_pool & pool, cudaStream_t main_stream+) {+ ggml_cuda_pool_alloc<float> dst_tmp(pool);+ ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);++ if (parallel_blocks > 1) {+ dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));+ dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));+ }++ constexpr int frag_m = (cols_per_block) == 8 && (D) % 32 == 0 ? 32 : 16;+ const dim3 block_dim(WARP_SIZE, nwarps, 1);+ const dim3 blocks_num(parallel_blocks*(Q->ne[1] + cols_per_block - 1) / cols_per_block, Q->ne[2], Q->ne[3]);+ const int shmem = 0;++ float scale;+ memcpy(&scale, KQV->op_params, sizeof(float));++ flash_attn_ext_f16<D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t>+ <<<blocks_num, block_dim, shmem, main_stream>>> (+ (const char *) Q->data,+ (const char *) K->data,+ (const char *) V->data,+ mask ? ((const char *) mask->data) : nullptr,+ (parallel_blocks) == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,+ scale,+ Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],+ K->ne[0], K->ne[1], K->ne[2], K->ne[3],+ mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,+ Q->nb[1], Q->nb[2], Q->nb[3],+ K->nb[1], K->nb[2], K->nb[3],+ KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]+ );+ CUDA_CHECK(cudaGetLastError());++ if ((parallel_blocks) == 1) {+ return;+ }++ const dim3 block_dim_combine(D, 1, 1);+ const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);+ const int shmem_combine = 0;++ flash_attn_combine_results<D, parallel_blocks>+ <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>+ (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);+ CUDA_CHECK(cudaGetLastError());+}++template <int D, int cols_per_block, int nwarps, typename KQ_acc_t> void launch_fattn_f16(+ const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,+ const int nsm, ggml_cuda_pool & pool, cudaStream_t main_stream+) {+ const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3];++ if (4*blocks_num_pb1 < 2*nsm) {+ launch_fattn_f16_impl<D, cols_per_block, nwarps, 4, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);+ return;+ }+ if (2*blocks_num_pb1 < 2*nsm) {+ launch_fattn_f16_impl<D, cols_per_block, nwarps, 2, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);+ return;+ }+ launch_fattn_f16_impl<D, cols_per_block, nwarps, 1, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);+}++void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * Q = dst->src[0];+ const ggml_tensor * K = dst->src[1];+ const ggml_tensor * V = dst->src[2];++ const ggml_tensor * mask = dst->src[3];++ ggml_tensor * KQV = dst;++ GGML_ASSERT(Q->type == GGML_TYPE_F32);+ GGML_ASSERT(K->type == GGML_TYPE_F16);+ GGML_ASSERT(V->type == GGML_TYPE_F16);+ GGML_ASSERT(KQV->type == GGML_TYPE_F32);++ GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);+ GGML_ASSERT(!mask || mask->ne[1] >= GGML_PAD(Q->ne[1], 16) &&+ "the Flash-Attention CUDA kernel requires the mask to be padded to 16 and at least n_queries big");++ GGML_ASSERT(K->ne[1] % FATTN_KQ_STRIDE == 0 && "Incorrect KV cache padding.");++ ggml_cuda_set_device(ctx.device);++ const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;++ const int32_t precision = KQV->op_params[1];++ if (precision != GGML_PREC_DEFAULT) {+ if (Q->ne[1] <= 32 || Q->ne[0] > 128) {+ constexpr int cols_per_block = 16;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ } else {+ constexpr int cols_per_block = 32;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ // case 256:+ // launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ // break;+ default:+ GGML_ASSERT(false);+ break;+ }+ }+ return;+ }++ if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0) {+ constexpr int parallel_blocks = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_vec_f16< 64, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());+ break;+ case 128:+ launch_fattn_vec_f16<128, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());+ break;+ case 256:+ launch_fattn_vec_f16<256, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ return;+ }++ if (Q->ne[1] <= 8 && Q->ne[0] % WARP_SIZE == 0) {+ constexpr int cols_per_block = 8;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ return;+ }++ if (Q->ne[1] <= 32) {+ constexpr int cols_per_block = 16;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ return;+ }++ constexpr int cols_per_block = 32;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ return;+}+template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>static __global__ void k_get_rows(const void * src0, const int32_t * src1, dst_t * dst,@@ -7924,13 +8878,14 @@ void ggml_cuda_op_mul_mat_vec_q(GGML_UNUSED(src1_padded_row_size);}+template <int block_size>static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {- const int row = blockIdx.x;+ const int row = blockIdx.x*blockDim.y + threadIdx.y;const int tid = threadIdx.x;float2 mean_var = make_float2(0.f, 0.f);- for (int col = tid; col < ncols; col += WARP_SIZE) {+ for (int col = tid; col < ncols; col += block_size) {const float xi = x[row*ncols + col];mean_var.x += xi;mean_var.y += xi * xi;@@ -7938,16 +8893,28 @@ static __global__ void norm_f32(const float * x, float * dst, const int ncols, c// sum up partial sumsmean_var = warp_reduce_sum(mean_var);+ if (block_size > WARP_SIZE) {+ __shared__ float2 s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = mean_var;+ }+ __syncthreads();+ mean_var = s_sum[lane_id];+ mean_var = warp_reduce_sum(mean_var);+ }const float mean = mean_var.x / ncols;const float var = mean_var.y / ncols - mean * mean;const float inv_std = rsqrtf(var + eps);- for (int col = tid; col < ncols; col += WARP_SIZE) {+ for (int col = tid; col < ncols; col += block_size) {dst[row*ncols + col] = (x[row*ncols + col] - mean) * inv_std;}}+template <int block_size>static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {// blockIdx.x: num_groups idx// threadIdx.x: block_size idx@@ -7962,62 +8929,117 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int grfloat tmp = 0.0f; // partial sum for thread in warp- for (int j = start; j < end; j += WARP_SIZE) {+ for (int j = start; j < end; j += block_size) {tmp += x[j];}tmp = warp_reduce_sum(tmp);+ if (block_size > WARP_SIZE) {+ __shared__ float s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = tmp;+ }+ __syncthreads();+ tmp = s_sum[lane_id];+ tmp = warp_reduce_sum(tmp);+ }float mean = tmp / group_size;tmp = 0.0f;- for (int j = start; j < end; j += WARP_SIZE) {+ for (int j = start; j < end; j += block_size) {float xi = x[j] - mean;dst[j] = xi;tmp += xi * xi;}tmp = warp_reduce_sum(tmp);+ if (block_size > WARP_SIZE) {+ __shared__ float s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = tmp;+ }+ __syncthreads();+ tmp = s_sum[lane_id];+ tmp = warp_reduce_sum(tmp);+ }float variance = tmp / group_size;float scale = rsqrtf(variance + eps);- for (int j = start; j < end; j += WARP_SIZE) {+ for (int j = start; j < end; j += block_size) {dst[j] *= scale;}}-static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps, int nrows) {- const int row = blockIdx.x;+template <int block_size>+static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {+ const int row = blockIdx.x*blockDim.y + threadIdx.y;const int tid = threadIdx.x;float tmp = 0.0f; // partial sum for thread in warp- for (int col = tid; col < ncols; col += WARP_SIZE) {+ for (int col = tid; col < ncols; col += block_size) {const float xi = x[row*ncols + col];tmp += xi * xi;}- const float mean = warp_reduce_sum(tmp) / ncols;+ // sum up partial sums+ tmp = warp_reduce_sum(tmp);+ if (block_size > WARP_SIZE) {+ __shared__ float s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = tmp;+ }+ __syncthreads();+ tmp = s_sum[lane_id];+ tmp = warp_reduce_sum(tmp);+ }++ const float mean = tmp / ncols;const float scale = rsqrtf(mean + eps);- for (int col = tid; col < ncols; col += WARP_SIZE) {+ for (int col = tid; col < ncols; col += block_size) {dst[row*ncols + col] = scale * x[row*ncols + col];}}static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {GGML_ASSERT(ncols % WARP_SIZE == 0);- norm_f32<<<nrows, WARP_SIZE, 0, stream>>>(x, dst, ncols, eps);+ if (ncols < 1024) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ } else {+ const dim3 block_dims(1024, 1, 1);+ norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ }}static void group_norm_f32_cuda(const float * x, float * dst, const int num_groups, const int group_size, const int ne_elements, cudaStream_t stream) {static const float eps = 1e-6f;- group_norm_f32<<<num_groups, WARP_SIZE, 0, stream>>>(x, dst, group_size, ne_elements, eps);+ if (group_size < 1024) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);+ } else {+ const dim3 block_dims(1024, 1, 1);+ group_norm_f32<1024><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);+ }}static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {GGML_ASSERT(ncols % WARP_SIZE == 0);- rms_norm_f32<<<nrows, WARP_SIZE, 0, stream>>>(x, dst, ncols, eps, nrows);+ if (ncols < 1024) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ rms_norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ } else {+ const dim3 block_dims(1024, 1, 1);+ rms_norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ }}void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {@@ -8593,8 +9615,18 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {CUDA_CHECK(cudaGetLastError());}-template <bool vals_smem, int ncols_template, int block_size_template>-static __global__ void soft_max_f32(const float * x, const float * mask, const float * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {+template <typename T>+static __device__ __forceinline__ float t2f32(T val) {+ return (float) val;+}++template <>+__device__ float __forceinline__ t2f32<half>(half val) {+ return __half2float(val);+}++template <bool vals_smem, int ncols_template, int block_size_template, typename T>+static __global__ void soft_max_f32(const float * x, const T * mask, const T * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {const int ncols = ncols_template == 0 ? ncols_par : ncols_template;const int tid = threadIdx.x;@@ -8621,7 +9653,7 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const fextern __shared__ float data_soft_max_f32[];float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication// shared memory buffer to cache values between iterations:- float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + rowx*ncols;+ float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + (int64_t)rowx*ncols;float max_val = -INFINITY;@@ -8633,10 +9665,10 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const fbreak;}- const int ix = rowx*ncols + col;- const int iy = rowy*ncols + col;+ const int64_t ix = (int64_t)rowx*ncols + col;+ const int64_t iy = (int64_t)rowy*ncols + col;- const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + (pos ? slope*pos[col] : 0.0f);+ const float val = x[ix]*scale + (mask ? t2f32(mask[iy]) : 0.0f) + (pos ? slope*t2f32(pos[col]) : 0.0f);vals[col] = val;max_val = max(max_val, val);@@ -8702,12 +9734,13 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const freturn;}- const int idst = rowx*ncols + col;+ const int64_t idst = (int64_t)rowx*ncols + col;dst[idst] = vals[col] * inv_sum;}}-static void soft_max_f32_cuda(const float * x, const float * mask, const float * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {+template<typename T>+static void soft_max_f32_cuda(const float * x, const T * mask, const T * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {int nth = WARP_SIZE;while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;const dim3 block_dims(nth, 1, 1);@@ -8760,15 +9793,19 @@ static void soft_max_f32_cuda(const float * x, const float * mask, const float *void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];+ const ggml_tensor * src2 = dst->src[2];+const float * src0_d = (const float *)src0->data;- const float * src1_d = src1 ? (const float *)src1->data : nullptr;+ const void * src1_d = src1 ? (const void *)src1->data : nullptr;+float * dst_d = (float *)dst->data;cudaStream_t stream = ctx.stream();GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);- GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional+ GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F16 || src2->type == GGML_TYPE_F32); // src2 contains positions and it is optionalconst int64_t ne00 = src0->ne[0];const int64_t nrows_x = ggml_nrows(src0);@@ -8781,16 +9818,27 @@ void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));// positions tensor- float * src2_dd = nullptr;+ void * src2_d = nullptr;- ggml_tensor * src2 = dst->src[2];const bool use_src2 = src2 != nullptr;if (use_src2) {- src2_dd = (float *)src2->data;+ src2_d = (void *)src2->data;}- soft_max_f32_cuda(src0_d, src1_d, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);++ if (use_f16) {+ const half * src1_dd = (const half *)src1_d;+ const half * src2_dd = (const half *)src2_d;++ soft_max_f32_cuda(src0_d, src1_dd, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);+ } else {+ const float * src1_dd = (const float *)src1_d;+ const float * src2_dd = (const float *)src2_d;++ soft_max_f32_cuda(src0_d, src1_dd, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);+ }}static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {@@ -9167,23 +10215,6 @@ void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {#include "ggml.h"#include "ggml-backend-impl.h"--#include <algorithm>-#include <array>-#include <atomic>-#include <cinttypes>-#include <cstddef>-#include <cstdint>-#include <float.h>-#include <limits>-#include <map>-#include <memory>-#include <mutex>-#include <stdint.h>-#include <stdio.h>-#include <string>-#include <vector>-static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");[[noreturn]]@@ -9284,6 +10315,7 @@ static ggml_cuda_device_info ggml_cuda_init() {info.devices[id].cc = 100*prop.major + 10*prop.minor;#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)info.devices[id].smpb = prop.sharedMemPerBlock;+ info.devices[id].nsm = prop.multiProcessorCount;}for (int id = 0; id < info.device_count; ++id) {@@ -11434,6 +12466,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggcase GGML_OP_ARGSORT:ggml_cuda_op_argsort(ctx, dst);break;+ case GGML_OP_FLASH_ATTN_EXT:+ ggml_cuda_flash_attn_ext(ctx, dst);+ break;default:return false;}@@ -11708,6 +12743,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, conscase GGML_OP_ARANGE:case GGML_OP_TIMESTEP_EMBEDDING:case GGML_OP_LEAKY_RELU:+ case GGML_OP_FLASH_ATTN_EXT:return true;default:return false;diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex ee24f60b..90870cb4 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -315,7 +315,7 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b)#endif // defined(__ARM_NEON)-#if defined(__ARM_NEON) && !defined(__MSC_VER)+#if defined(__ARM_NEON) && !defined(_MSC_VER)#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex ed5408a4..d32007a7 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -29,6 +29,7 @@ static const struct ggml_backend_api *g_backend;#define ggml_type_size g_backend->ggml_type_size#define ggml_guid_matches g_backend->ggml_guid_matches#define ggml_is_empty g_backend->ggml_is_empty+#define ggml_are_same_shape g_backend->ggml_are_same_shapevoid ggml_metal_link(const struct ggml_backend_api *backend_api) {g_backend = backend_api;@@ -73,8 +74,10 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_GELU_QUICK_4,GGML_METAL_KERNEL_TYPE_SILU,GGML_METAL_KERNEL_TYPE_SILU_4,- GGML_METAL_KERNEL_TYPE_SOFT_MAX,- GGML_METAL_KERNEL_TYPE_SOFT_MAX_4,+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16,+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4,+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32,+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4,GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF,GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8,GGML_METAL_KERNEL_TYPE_GET_ROWS_F32,@@ -204,6 +207,14 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC,GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128,+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256,GGML_METAL_KERNEL_TYPE_CPY_F32_F16,GGML_METAL_KERNEL_TYPE_CPY_F32_F32,GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0,@@ -472,7 +483,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {}/*- GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \+ GGML_METAL_LOG_INFO("%s: loaded %-40s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \(int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \(int) kernel->pipeline.threadExecutionWidth); \*/@@ -488,172 +499,182 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {return NULL; \} \} else { \- GGML_METAL_LOG_WARN("%s: skipping %-32s (not supported)\n", __func__, "kernel_"#name); \+ GGML_METAL_LOG_WARN("%s: skipping %-40s (not supported)\n", __func__, "kernel_"#name); \}// simd_sum and simd_max requires MTLGPUFamilyApple7- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CLAMP, clamp, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_4, gelu_4, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK_4, gelu_quick_4, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU_4, silu_4, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX, soft_max, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_4, soft_max_4, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M, get_rows_iq1_m, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32, mul_mv_iq1_m_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, mul_mv_iq4_xs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, ctx->support_simdgroup_reduction);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32, mul_mm_id_iq1_m_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CLAMP, clamp, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_4, gelu_4, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK_4, gelu_quick_4, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU_4, silu_4, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16, soft_max_f16, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4, soft_max_f16_4, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32, soft_max_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4, soft_max_f32_4, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M, get_rows_iq1_m, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32, mul_mv_iq1_m_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, mul_mv_iq4_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32, mul_mm_id_iq1_m_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64, flash_attn_ext_f16_h64, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80, flash_attn_ext_f16_h80, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96, flash_attn_ext_f16_h96, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112, flash_attn_ext_f16_h112, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128, flash_attn_ext_f16_h128, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, flash_attn_ext_f16_h256, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128, flash_attn_ext_vec_f16_h128, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, flash_attn_ext_vec_f16_h256, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);}[metal_library release];@@ -772,6 +793,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constcase GGML_OP_TIMESTEP_EMBEDDING:case GGML_OP_ARGSORT:case GGML_OP_LEAKY_RELU:+ case GGML_OP_FLASH_ATTN_EXT:return true;case GGML_OP_MUL_MAT:case GGML_OP_MUL_MAT_ID:@@ -1355,20 +1377,33 @@ static enum ggml_status ggml_metal_graph_compute(} break;case GGML_OP_SOFT_MAX:{+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);+ GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F16 || src2->type == GGML_TYPE_F32);+int nth = 32; // SIMD widthid<MTLComputePipelineState> pipeline = nil;+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);+if (ne00%4 == 0) {while (nth < ne00/4 && nth < 256) {nth *= 2;}- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_4].pipeline;+ if (use_f16) {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4].pipeline;+ } else {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline;+ }} else {while (nth < ne00 && nth < 1024) {nth *= 2;}- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline;+ if (use_f16) {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16].pipeline;+ } else {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32].pipeline;+ }}float scale;@@ -2532,6 +2567,161 @@ static enum ggml_status ggml_metal_graph_compute([encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];} break;+ case GGML_OP_FLASH_ATTN_EXT:+ {+ GGML_ASSERT(ne00 % 4 == 0);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);++ struct ggml_tensor * src3 = gf->nodes[i]->src[3];++ GGML_ASSERT(ggml_are_same_shape(src1, src2));+ GGML_ASSERT(src3);++ size_t offs_src3 = 0;++ id<MTLBuffer> id_src3 = src3 ? ggml_metal_get_buffer(src3, &offs_src3) : nil;++ GGML_ASSERT(!src3 || src3->type == GGML_TYPE_F16);+ GGML_ASSERT(!src3 || src3->ne[1] >= GGML_PAD(src0->ne[1], 8) &&+ "the Flash-Attention Metal kernel requires the mask to be padded to 8 and at least n_queries big");++ const int64_t ne30 = src3 ? src3->ne[0] : 0; GGML_UNUSED(ne30);+ const int64_t ne31 = src3 ? src3->ne[1] : 0;+ const int64_t ne32 = src3 ? src3->ne[2] : 0; GGML_UNUSED(ne32);+ const int64_t ne33 = src3 ? src3->ne[3] : 0; GGML_UNUSED(ne33);++ const uint64_t nb30 = src3 ? src3->nb[0] : 0; GGML_UNUSED(nb30);+ const uint64_t nb31 = src3 ? src3->nb[1] : 0;+ const uint64_t nb32 = src3 ? src3->nb[2] : 0; GGML_UNUSED(nb32);+ const uint64_t nb33 = src3 ? src3->nb[3] : 0; GGML_UNUSED(nb33);++ const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);++ float scale;+ memcpy(&scale, dst->op_params, sizeof(float));++ id<MTLComputePipelineState> pipeline = nil;++ bool use_vec_kernel = false;++ if (ne01 >= 4 || (ne00%128 != 0)) {+ switch (ne00) {+ case 64: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64 ].pipeline; break;+ case 80: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80 ].pipeline; break;+ case 96: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96 ].pipeline; break;+ case 112: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112].pipeline; break;+ case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128].pipeline; break;+ case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256].pipeline; break;+ default:+ {+ GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);+ GGML_METAL_LOG_ERROR("add template specialization for this size\n");+ GGML_ASSERT(false && "add template specialization for this size");+ }+ }+ } else {+ use_vec_kernel = true;++ switch (ne00) {+ case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128].pipeline; break;+ case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256].pipeline; break;+ default:+ {+ GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);+ GGML_METAL_LOG_ERROR("add template specialization for this size\n");+ GGML_ASSERT(false && "add template specialization for this size");+ }+ }+ }++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];+ [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:4];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:5];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:6];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:7];+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:8];+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:10];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:11];+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:12];+ [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:13];+ [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:14];+ [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:15];+ [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:16];+ [encoder setBytes:&nb10 length:sizeof(uint64_t) atIndex:17];+ [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:18];+ [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:19];+ [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:20];+ [encoder setBytes:&ne31 length:sizeof( int64_t) atIndex:21];+ [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:22];+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:23];+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:24];+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:25];+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:26];+ [encoder setBytes:&scale length:sizeof( float) atIndex:27];++ if (!use_vec_kernel) {+ // half8x8 kernel+ const int64_t nqptg = 8; // queries per threadgroup !! sync with kernel template arguments !!+ const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!++ GGML_ASSERT(nqptg <= 32);+ GGML_ASSERT(nqptg % 8 == 0);+ GGML_ASSERT(ncpsg % 32 == 0);++ int64_t nsgmax = 2;++ while (true) {+ const size_t smem = nqptg*(ne00 + 2*nsgmax*(ncpsg + nqptg))*(sizeof(float)/2);+ if (smem > ctx->device.maxThreadgroupMemoryLength) {+ break;+ }+ nsgmax *= 2;+ }+ nsgmax /= 2;++ // simdgroups per threadgroup (a.k.a. warps)+ const int64_t nsg = ne01 <= nqptg ? MAX(4, MIN(nsgmax, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32))) : 4;++ const size_t smem = nqptg*(ne00 + 2*nsg*(ncpsg + nqptg))*(sizeof(float)/2);++ //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);+ GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);++ [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];++ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];+ } else {+ // half1x4 kernel+ const int64_t nqptg = 1; // queries per threadgroup !! sync with kernel template arguments !!+ const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!++ GGML_ASSERT(nqptg <= 32);+ GGML_ASSERT(nqptg % 1 == 0);+ GGML_ASSERT(ncpsg % 32 == 0);++ // simdgroups per threadgroup (a.k.a. warps)+ const int64_t nsgt = MAX(2, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32));++ int64_t nsg = 1;+ while (nsg <= nsgt) {+ nsg *= 2;+ }+ nsg /= 2;++ const size_t smem = (nqptg*(ne00 + 2*nsg*(ncpsg + nqptg)) + nsg*ne00)*(sizeof(float)/2);++ //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);+ GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);+ [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];++ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];+ }+ } break;case GGML_OP_DUP:case GGML_OP_CPY:case GGML_OP_CONT:@@ -2619,6 +2809,11 @@ static enum ggml_status ggml_metal_graph_compute(MTLCommandBufferStatus status = [command_buffer status];if (status != MTLCommandBufferStatusCompleted) {GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);+ if (status == MTLCommandBufferStatusError) {+ NSString * error_code = [command_buffer error].localizedDescription;+ GGML_METAL_LOG_INFO("error: %s\n", [error_code UTF8String]);+ }+return GGML_STATUS_FAILED;}}@@ -2735,10 +2930,13 @@ GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backeUNUSED(buft);}-static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device) {+static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device, size_t size_aligned) {+#ifndef GGML_METAL_NDEBUG#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)if (@available(macOS 10.12, iOS 16.0, *)) {- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",+ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f / %8.2f)",+ __func__,+ size_aligned / 1024.0 / 1024.0,device.currentAllocatedSize / 1024.0 / 1024.0,device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);@@ -2748,10 +2946,15 @@ static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device) {GGML_METAL_LOG_INFO("\n");}} else {- GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);+ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f)\n",+ __func__,+ size_aligned / 1024.0 / 1024.0,+ device.currentAllocatedSize / 1024.0 / 1024.0);}+#endif#endifUNUSED(device);+ UNUSED(size_aligned);}GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {@@ -2785,8 +2988,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffreturn NULL;}- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);- ggml_backend_metal_log_allocated_size(device);+ //ggml_backend_metal_log_allocated_size(device, size_aligned);return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);}@@ -2873,7 +3075,7 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,return false;}- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);+ ggml_backend_metal_log_allocated_size(device, size_aligned);++ctx->n_buffers;} else {@@ -2896,7 +3098,8 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,return false;}- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, offs = %12ld", __func__, size_step_aligned / 1024.0 / 1024.0, i);+ ggml_backend_metal_log_allocated_size(device, size_step_aligned);+if (i + size_step < size) {GGML_METAL_LOG_INFO("\n");}@@ -2905,8 +3108,6 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,}}- ggml_backend_metal_log_allocated_size(device);-return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);}diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metalindex e666ea56..382e4278 100644--- a/llama.cpp/ggml-metal.metal+++ b/llama.cpp/ggml-metal.metal@@ -2206,11 +2206,12 @@ kernel void kernel_sum_rows(dst_row[0] = row_sum;}+template<typename T>kernel void kernel_soft_max(- device const float * src0,- device const float * src1,- device const float * src2,- device float * dst,+ device const char * src0,+ device const char * src1,+ device const char * src2,+ device char * dst,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -2229,10 +2230,10 @@ kernel void kernel_soft_max(const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);- device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;- device const float * pmask = src1 != src0 ? src1 + i01*ne00 : nullptr;- device const float * ppos = src2 != src0 ? src2 : nullptr;- device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;+ device const float * psrc0 = (device const float *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);+ device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00 : nullptr;+ device const T * ppos = src2 != src0 ? (device const T *) src2 : nullptr;+ device float * pdst = (device float *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);float slope = 0.0f;@@ -2310,11 +2311,12 @@ kernel void kernel_soft_max(}}+template<typename T>kernel void kernel_soft_max_4(- device const float * src0,- device const float * src1,- device const float * src2,- device float * dst,+ device const char * src0,+ device const char * src1,+ device const char * src2,+ device char * dst,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -2333,10 +2335,10 @@ kernel void kernel_soft_max_4(const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);- device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);- device const float4 * pmask = src1 != src0 ? (device const float4 *)(src1 + i01*ne00) : nullptr;- device const float4 * ppos = src2 != src0 ? (device const float4 *)(src2) : nullptr;- device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);+ device const float4 * psrc4 = (device const float4 *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;+ device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00/4 : nullptr;+ device const T * ppos = src2 != src0 ? (device const T *) src2 : nullptr;+ device float4 * pdst4 = (device float4 *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;float slope = 0.0f;@@ -2353,7 +2355,7 @@ kernel void kernel_soft_max_4(float4 lmax4 = -INFINITY;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (float4)((pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)));}const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));@@ -2379,7 +2381,7 @@ kernel void kernel_soft_max_4(// parallel sumfloat4 lsum4 = 0.0f;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (float4)((pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f))) - max_val);lsum4 += exp_psrc4;pdst4[i00] = exp_psrc4;}@@ -2416,6 +2418,14 @@ kernel void kernel_soft_max_4(}}+typedef decltype(kernel_soft_max<float>) kernel_soft_max_t;+typedef decltype(kernel_soft_max_4<float4>) kernel_soft_max_4_t;++template [[host_name("kernel_soft_max_f16")]] kernel kernel_soft_max_t kernel_soft_max<half>;+template [[host_name("kernel_soft_max_f32")]] kernel kernel_soft_max_t kernel_soft_max<float>;+template [[host_name("kernel_soft_max_f16_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4<half4>;+template [[host_name("kernel_soft_max_f32_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4<float4>;+kernel void kernel_diag_mask_inf(device const float * src0,device float * dst,@@ -3938,6 +3948,632 @@ kernel void kernel_leaky_relu_f32(dst[tpig] = src0[tpig] > 0.0f ? src0[tpig] : src0[tpig] * slope;}+typedef void (flash_attn_ext_f16_t)(+ device const char * q,+ device const char * k,+ device const char * v,+ device const char * mask,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne31,+ constant uint64_t & nb31,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant float & scale,+ threadgroup half * shared,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]],+ ushort tiisg[[thread_index_in_simdgroup]],+ ushort sgitg[[simdgroup_index_in_threadgroup]]);++// ref: https://arxiv.org/pdf/2307.08691.pdf+template<int64_t D, int64_t Q = 8, int64_t C = 32> // head size, queries per threadgroup, cache items per threadgroup+kernel void kernel_flash_attn_ext_f16(+ device const char * q,+ device const char * k,+ device const char * v,+ device const char * mask,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne31,+ constant uint64_t & nb31,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant float & scale,+ threadgroup half * shared [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]],+ ushort tiisg[[thread_index_in_simdgroup]],+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {+ const short nsg = ntg.y; // number of simdgroups++ const short iq3 = tgpig[2];+ const short iq2 = tgpig[1];+ const short iq1 = tgpig[0]*Q;++ const short D4 = D/4;+ const short D8 = D/8;+ const short Q8 = Q/8;+ const short NW = N_SIMDWIDTH;+ const short SH = (C + Q); // shared memory per simdgroup in (half)++ const short T = D + 2*nsg*SH; // shared memory size per query in (half)+ const short TF = T/2; // shared memory size per query in (float)+ const short T4 = T/4; // shared memory size per query in (half4)++ threadgroup half * sq = (threadgroup half *) (shared + 0*D); // holds the query data+ threadgroup half4 * sq4 = (threadgroup half4 *) (shared + 0*D); // same as above but in half4+ threadgroup float * ss = (threadgroup float *) (shared + 2*sgitg*SH + 1*D); // scratch buffer for attention and diagonal matrix++ // store the result for all queries in local memory in 8x8 matrices (the O matrix from the paper)+ simdgroup_half8x8 lo[D8];++ // load heads from Q to shared memory+ for (short j = sgitg; j < Q; j += nsg) {+ device const float4 * q4 = (device const float4 *) ((device const char *) q + ((iq1 + j)*nb01 + iq2*nb02 + iq3*nb03));++ for (short i = tiisg; i < D4; i += NW) {+ if (iq1 + j < ne01) {+ sq4[j*T4 + i] = (half4) q4[i];+ } else {+ sq4[j*T4 + i] = 0.0h;+ }+ }+ }++ // zero out lo+ for (short i = 0; i < D8; ++i) {+ lo[i] = make_filled_simdgroup_matrix<half, 8>(0.0h);+ }++ // zero out shared memory SH+ for (short j = 0; j < Q; ++j) {+ for (short i = tiisg; i < SH; i += NW) {+ ss[j*TF + i] = 0.0f;+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ {+ float S[Q] = { [0 ... Q-1] = 0.0h };+ float M[Q] = { [0 ... Q-1] = -FLT_MAX/2 };++ // assume K and V are same shape+ const short ne22 = ne12;+ const short ne23 = ne13;++ const uint nb21 = nb11;+ const uint nb22 = nb12;+ const uint nb23 = nb13;++ // broadcast+ const short rk2 = ne02/ne12;+ const short rk3 = ne03/ne13;++ const short rv2 = ne02/ne22;+ const short rv3 = ne03/ne23;++ // k indices+ const short ik2 = iq2/rk2;+ const short ik3 = iq3/rk3;++ // v indices+ const short iv2 = iq2/rv2;+ const short iv3 = iq3/rv3;++ // load the queries from shared memory into local memory+ simdgroup_half8x8 mq[D8];++ for (short i = 0; i < D8; ++i) {+ simdgroup_load(mq[i], sq + i*8, T);+ }++ // pointer to the mask+ device const half * mp = (device const half *) (mask + iq1*nb31);++ // prepare diagonal scale matrix+ simdgroup_float8x8 mscale(scale);++ // loop over the KV cache+ // each simdgroup handles blocks of Q rows and C columns+ for (int ic0 = 0; ic0 < ne11; ic0 += C*nsg) {+ const int ic = ic0 + C*sgitg;+ if (ic >= ne11) {+ break;+ }++ // Q*K^T+ {+ for (short cc = 0; cc < C/8; ++cc) {+ simdgroup_float8x8 mqk = make_filled_simdgroup_matrix<float, 8>(0.h);++ device const half * pk = (device const half *) ((device const char *) k + ((ic + 8*cc)*nb11 + ik2*nb12 + ik3*nb13));++ for (short i = 0; i < D8; ++i) {+ simdgroup_half8x8 mk;+ simdgroup_load(mk, pk + i*8, nb11/sizeof(half), 0, true); // transpose++ simdgroup_multiply_accumulate(mqk, mq[i], mk, mqk);+ }++ // mqk = mqk*scale + mask+ simdgroup_half8x8 mm;+ simdgroup_load(mm, mp + ic + 8*cc, nb31/sizeof(half), 0, false);+ simdgroup_multiply_accumulate(mqk, mqk, mscale, mm);++ simdgroup_store(mqk, ss + 8*cc, TF, 0, false);+ }+ }++ // used to detect blocks full of -INF+ float smax = -INFINITY;++ // online softmax+ {+ float ms[Q];++ for (short j = 0; j < Q; ++j) {+ const short p = tiisg;++ const float m = M[j];+ const float s = ss[j*TF + p];++ smax = simd_max(max(smax, s));+ M[j] = simd_max(max(M[j], s));++ ms[j] = exp(m - M[j]);+ const float vs = exp(s - M[j]);++ S[j] = S[j]*ms[j] + simd_sum(vs);++ // the P matrix from the paper (Q rows, C columns)+ ss[j*TF + p] = vs;+ }++ // create a QxQ diagonal matrix for rescaling the output+ if (tiisg < Q) {+ ss[tiisg*TF + C + tiisg] = ms[tiisg];+ }+ }++ // skip -INF blocks+ if (smax == -INFINITY) {+ continue;+ }++ // O = diag(ms)*O+ {+ simdgroup_float8x8 mm;+ simdgroup_load(mm, ss + C, TF, 0, false);++ for (short i = 0; i < D8; ++i) {+ simdgroup_multiply(lo[i], mm, lo[i]);+ }+ }++ // O = O + (Q*K^T)*V+ {+ for (short cc = 0; cc < C/8; ++cc) {+ device const half * pv = (device const half *) ((device const char *) v + ((ic + 8*cc)*nb21 + iv2*nb22 + iv3*nb23));++ for (short i = 0; i < D8; ++i) {+ simdgroup_half8x8 mk;+ simdgroup_load(mk, pv + i*8, nb21/sizeof(half), 0, false);++ simdgroup_float8x8 mv;+ simdgroup_load(mv, ss + 8*cc, TF, 0, false);++ simdgroup_multiply_accumulate(lo[i], mv, mk, lo[i]);+ }+ }+ }+ }++ // these are needed for reducing the results from the simdgroups (reuse the ss buffer)+ for (short j = 0; j < Q; ++j) {+ if (tiisg == 0) {+ ss[j*TF + 0] = S[j];+ ss[j*TF + 1] = M[j];+ }+ }+ }++ // reduce the warps sequentially+ for (short sg = 1; sg < nsg; ++sg) {+ float S = { 0.0h };+ float M = { -FLT_MAX/2 };++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // each simdgroup stores its output to shared memory, reusing sq+ if (sgitg == sg) {+ for (short i = 0; i < D8; ++i) {+ simdgroup_store(lo[i], sq + i*8, T, 0, false);+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // the first simdgroup accumulates the results from the other simdgroups+ if (sgitg == 0) {+ for (short j = 0; j < Q; ++j) {+ const float S0 = ss[j*TF + 0];+ const float S1 = ss[j*TF + sg*SH + 0];++ const float M0 = ss[j*TF + 1];+ const float M1 = ss[j*TF + sg*SH + 1];++ M = max(M0, M1);++ const float ms0 = exp(M0 - M);+ const float ms1 = exp(M1 - M);++ S = S0*ms0 + S1*ms1;++ if (tiisg == 0) {+ ss[j*TF + 0] = S;+ ss[j*TF + 1] = M;++ ss[j*TF + C + j ] = ms0;+ ss[j*TF + C + j + sg*SH] = ms1;+ }+ }++ // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1+ {+ simdgroup_half8x8 t;+ simdgroup_float8x8 ms0;+ simdgroup_float8x8 ms1;++ simdgroup_load(ms0, ss + C, TF, 0, false);+ simdgroup_load(ms1, ss + C + sg*SH, TF, 0, false);++ for (short i = 0; i < D8; ++i) {+ simdgroup_load (t, sq + i*8, T, 0, false);+ simdgroup_multiply(t, ms1, t);++ simdgroup_multiply_accumulate(lo[i], ms0, lo[i], t);+ }+ }+ }+ }++ // store result to shared memory (reuse sq)+ if (sgitg == 0) {+ for (short i = 0; i < D8; ++i) {+ simdgroup_store(lo[i], sq + i*8, T, 0, false);+ }+ }++ device float4 * dst4 = (device float4 *) dst;++ // final rescale with 1/S and store to global memory+ if (sgitg == 0) {+ for (short j = 0; j < Q && iq1 + j < ne01; ++j) {+ const float S = ss[j*TF + 0];++ for (short i = tiisg; i < D4; i += NW) {+ dst4[(iq3*ne2*ne1 + iq2 + (iq1 + j)*ne1)*D4 + i] = (float4) sq4[j*T4 + i]/S;+ }+ }+ }+}++template [[host_name("kernel_flash_attn_ext_f16_h64" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<64>;+template [[host_name("kernel_flash_attn_ext_f16_h80" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<80>;+template [[host_name("kernel_flash_attn_ext_f16_h96" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<96>;+template [[host_name("kernel_flash_attn_ext_f16_h112")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<112>;+template [[host_name("kernel_flash_attn_ext_f16_h128")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<128>;+template [[host_name("kernel_flash_attn_ext_f16_h256")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<256>;++template<int64_t D, int64_t Q = 1, int64_t C = 32> // head size, queries per threadgroup, cache items per threadgroup+kernel void kernel_flash_attn_ext_vec_f16(+ device const char * q,+ device const char * k,+ device const char * v,+ device const char * mask,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne31,+ constant uint64_t & nb31,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant float & scale,+ threadgroup half * shared [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]],+ ushort tiisg[[thread_index_in_simdgroup]],+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {+ const short nsg = ntg.y; // number of simdgroups++ const short iq3 = tgpig[2];+ const short iq2 = tgpig[1];+ const short iq1 = tgpig[0];++ const short D4 = D/4;+ const short NW = N_SIMDWIDTH;+ const short SH = (C + Q); // shared memory per simdgroup in (half)++ const short T = D + 2*nsg*SH; // shared memory size per query in (half)++ //threadgroup half * sq = (threadgroup half *) (shared + 0*D); // holds the query data+ threadgroup half4 * sq4 = (threadgroup half4 *) (shared + 0*D); // same as above but in half4+ threadgroup float * ss = (threadgroup float *) (shared + 2*sgitg*SH + 1*D); // scratch buffer for attention and diagonal matrix+ threadgroup float4 * ss4 = (threadgroup float4 *) (shared + 2*sgitg*SH + 1*D); // same as above but in half4+ threadgroup half4 * sr4 = (threadgroup half4 *) (shared + sgitg*D + 1*T); // scratch buffer for the results++ // store the result for all queries in local memory in 8x8 matrices (the O matrix from the paper)+ half4 lo[D4/NW];++ // load heads from Q to shared memory+ device const float4 * q4 = (device const float4 *) ((device const char *) q + (iq1*nb01 + iq2*nb02 + iq3*nb03));++ for (short i = tiisg; i < D4; i += NW) {+ if (iq1 < ne01) {+ sq4[i] = (half4) q4[i];+ } else {+ sq4[i] = 0.0h;+ }+ }++ // zero out lo+ for (short i = tiisg; i < D4; i += NW) {+ lo[i/NW] = 0.0h;+ }++ // zero out shared memory SH+ for (short i = tiisg; i < SH/4; i += NW) {+ ss4[i] = 0.0h;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ {+ float S = { 0.0h };+ float M = { -FLT_MAX/2 };++ // assume K and V are same shape+ const short ne22 = ne12;+ const short ne23 = ne13;++ const uint nb21 = nb11;+ const uint nb22 = nb12;+ const uint nb23 = nb13;++ // broadcast+ const short rk2 = ne02/ne12;+ const short rk3 = ne03/ne13;++ const short rv2 = ne02/ne22;+ const short rv3 = ne03/ne23;++ // k indices+ const short ik2 = iq2 / rk2;+ const short ik3 = iq3 / rk3;++ // v indices+ const short iv2 = iq2 / rv2;+ const short iv3 = iq3 / rv3;++ // load the queries from shared memory into local memory+ half4 mq[D4];++ for (short ii = 0; ii < D4; ii += NW) {+ short i = ii + tiisg;+ mq[i] = sq4[i];+ }++ // pointer to the mask+ device const half4 * mp4 = (device const half4 *) (mask + iq1*nb31);++ // loop over the KV cache+ // each simdgroup handles blocks of Q rows and C columns+ for (int ic0 = 0; ic0 < ne11; ic0 += C*nsg) {+ const int ic = ic0 + C*sgitg;+ if (ic >= ne11) {+ break;+ }++ // Q*K^T+ {+#pragma unroll+ for (short cc = 0; cc < C/4; ++cc) {+ float4 mqk = { 0.0h };++ device const half4 * pk4 = (device const half4 *) ((device const char *) k + ((ic + 4*cc)*nb11 + ik2*nb12 + ik3*nb13));++#pragma unroll+ for (short ii = 0; ii < D4; ii += NW) {+ const short i = ii + tiisg;++ half4x4 mk;+ mk[0] = pk4[i + 0*(nb11/8)];+ mk[1] = pk4[i + 1*(nb11/8)];+ mk[2] = pk4[i + 2*(nb11/8)];+ mk[3] = pk4[i + 3*(nb11/8)];++ mqk += (float4) (mq[i] * mk);+ }++ // reduce the results from the threads in the simdgroup+ mqk += simd_shuffle_down(mqk, 16);+ mqk += simd_shuffle_down(mqk, 8);+ mqk += simd_shuffle_down(mqk, 4);+ mqk += simd_shuffle_down(mqk, 2);+ mqk += simd_shuffle_down(mqk, 1);++ // mqk = mqk*scale + mask+ if (tiisg == 0) {+ float4 mm = (float4) mp4[ic/4 + cc];+ mqk = mqk*scale + mm;++ ss4[cc] = mqk;+ }+ }+ }++ // online softmax+ {+ const short p = tiisg;++ const float m = M;+ const float s = ss[p];++ M = simd_max(max(M, s));++ const float ms = exp(m - M);+ const float vs = exp(s - M);++ S = S*ms + simd_sum(vs);++ // the P matrix from the paper (Q rows, C columns)+ ss[p] = vs;++ // O = diag(ms)*O+#pragma unroll+ for (short ii = 0; ii < D4; ii += NW) {+ const short i = ii + tiisg;+ lo[i/NW] *= ms;+ }+ }++ // O = O + (Q*K^T)*V+ {+#pragma unroll+ for (short cc = 0; cc < C/4; ++cc) {+ device const half4 * pv4 = (device const half4 *) ((device const char *) v + ((ic + 4*cc)*nb21 + iv2*nb22 + iv3*nb23));++#pragma unroll+ for (short ii = 0; ii < D4; ii += NW) {+ const short i = ii + tiisg;++ lo[i/NW] += pv4[i + 0*(nb21/8)] * ss[4*cc + 0];+ lo[i/NW] += pv4[i + 1*(nb21/8)] * ss[4*cc + 1];+ lo[i/NW] += pv4[i + 2*(nb21/8)] * ss[4*cc + 2];+ lo[i/NW] += pv4[i + 3*(nb21/8)] * ss[4*cc + 3];+ }+ }+ }++ }++ // these are needed for reducing the results from the simdgroups (reuse the ss buffer)+ if (tiisg == 0) {+ ss[0] = S;+ ss[1] = M;+ }+ }++ // store results to shared memory+ for (short ii = 0; ii < D4; ii += NW) {+ short i = ii + tiisg;+ sr4[i] = lo[ii/NW];+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // parallel reduce+ for (short r = nsg/2; r > 0; r >>= 1) {+ if (sgitg < r) {+ const float S0 = ss[ 0];+ const float S1 = ss[r*SH + 0];++ const float M0 = ss[ 1];+ const float M1 = ss[r*SH + 1];++ const float M = max(M0, M1);++ const float ms0 = exp(M0 - M);+ const float ms1 = exp(M1 - M);++ const float S = S0*ms0 + S1*ms1;++ if (tiisg == 0) {+ ss[0] = S;+ ss[1] = M;+ }++ // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1+ for (short ii = 0; ii < D4; ii += NW) {+ short i = ii + tiisg;+ sr4[i] = sr4[i]*ms0 + sr4[i + r*D4]*ms1;+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ device float4 * dst4 = (device float4 *) dst;++ // final rescale with 1/S and store to global memory+ if (sgitg == 0) {+ const float S = ss[0];++ for (short ii = 0; ii < D4; ii += NW) {+ short i = ii + tiisg;+ dst4[(iq3*ne2*ne1 + iq2 + (iq1)*ne1)*D4 + i] = (float4) sr4[i]/S;+ }+ }+}++template [[host_name("kernel_flash_attn_ext_vec_f16_h128")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_vec_f16<128>;+template [[host_name("kernel_flash_attn_ext_vec_f16_h256")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_vec_f16<256>;+kernel void kernel_cpy_f16_f16(device const half * src0,device half * dst,diff --git a/llama.cpp/ggml-quants-amd-avx.c b/llama.cpp/ggml-quants-amd-avx.cnew file mode 100644index 00000000..dc6ace8f--- /dev/null+++ b/llama.cpp/ggml-quants-amd-avx.c@@ -0,0 +1,100 @@+#ifdef __x86_64__+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_amd_avx+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_amd_avx+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_amd_avx+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_amd_avx+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_amd_avx+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_amd_avx+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_amd_avx+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_amd_avx+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_amd_avx+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_amd_avx+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_amd_avx+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_amd_avx+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_amd_avx+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_amd_avx+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_amd_avx+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_amd_avx+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_amd_avx+#define quantize_row_q4_0 quantize_row_q4_0_amd_avx+#define quantize_row_q4_1 quantize_row_q4_1_amd_avx+#define quantize_row_q5_0 quantize_row_q5_0_amd_avx+#define quantize_row_q5_1 quantize_row_q5_1_amd_avx+#define quantize_row_q8_0 quantize_row_q8_0_amd_avx+#define quantize_row_q8_1 quantize_row_q8_1_amd_avx+#define quantize_row_q2_K quantize_row_q2_K_amd_avx+#define quantize_row_q3_K quantize_row_q3_K_amd_avx+#define quantize_row_q4_K quantize_row_q4_K_amd_avx+#define quantize_row_q5_K quantize_row_q5_K_amd_avx+#define quantize_row_q6_K quantize_row_q6_K_amd_avx+#define quantize_row_q8_K quantize_row_q8_K_amd_avx+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_amd_avx+#define quantize_row_iq4_nl quantize_row_iq4_nl_amd_avx+#define quantize_row_iq4_xs quantize_row_iq4_xs_amd_avx+#define quantize_row_iq3_s quantize_row_iq3_s_amd_avx+#define quantize_row_iq2_s quantize_row_iq2_s_amd_avx+#define dequantize_row_q4_0 dequantize_row_q4_0_amd_avx+#define dequantize_row_q4_1 dequantize_row_q4_1_amd_avx+#define dequantize_row_q5_0 dequantize_row_q5_0_amd_avx+#define dequantize_row_q5_1 dequantize_row_q5_1_amd_avx+#define dequantize_row_q8_0 dequantize_row_q8_0_amd_avx+#define dequantize_row_q2_K dequantize_row_q2_K_amd_avx+#define dequantize_row_q3_K dequantize_row_q3_K_amd_avx+#define dequantize_row_q4_K dequantize_row_q4_K_amd_avx+#define dequantize_row_q5_K dequantize_row_q5_K_amd_avx+#define dequantize_row_q6_K dequantize_row_q6_K_amd_avx+#define dequantize_row_q8_K dequantize_row_q8_K_amd_avx+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_amd_avx+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_amd_avx+#define dequantize_row_iq2_s dequantize_row_iq2_s_amd_avx+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_amd_avx+#define dequantize_row_iq1_s dequantize_row_iq1_s_amd_avx+#define dequantize_row_iq1_m dequantize_row_iq1_m_amd_avx+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_amd_avx+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_amd_avx+#define dequantize_row_iq3_s dequantize_row_iq3_s_amd_avx+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_amd_avx+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_amd_avx+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_amd_avx+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_amd_avx+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_amd_avx+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_amd_avx+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_amd_avx+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_amd_avx+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_amd_avx+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_amd_avx+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_amd_avx+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_amd_avx+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_amd_avx+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_amd_avx+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_amd_avx+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_amd_avx+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_amd_avx+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_amd_avx+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_amd_avx+#define quantize_iq2_xxs quantize_iq2_xxs_amd_avx+#define quantize_iq2_xs quantize_iq2_xs_amd_avx+#define quantize_iq2_s quantize_iq2_s_amd_avx+#define quantize_iq3_xxs quantize_iq3_xxs_amd_avx+#define quantize_iq1_s quantize_iq1_s_amd_avx+#define quantize_iq1_m quantize_iq1_m_amd_avx+#define quantize_iq4_nl quantize_iq4_nl_amd_avx+#define quantize_iq4_xs quantize_iq4_xs_amd_avx+#define quantize_iq3_s quantize_iq3_s_amd_avx+#define quantize_q2_K quantize_q2_K_amd_avx+#define quantize_q3_K quantize_q3_K_amd_avx+#define quantize_q4_K quantize_q4_K_amd_avx+#define quantize_q5_K quantize_q5_K_amd_avx+#define quantize_q6_K quantize_q6_K_amd_avx+#define quantize_q4_0 quantize_q4_0_amd_avx+#define quantize_q4_1 quantize_q4_1_amd_avx+#define quantize_q5_0 quantize_q5_0_amd_avx+#define quantize_q5_1 quantize_q5_1_amd_avx+#define quantize_q8_0 quantize_q8_0_amd_avx+#define iq2xs_init_impl iq2xs_init_impl_amd_avx+#define iq2xs_free_impl iq2xs_free_impl_amd_avx+#define iq3xs_init_impl iq3xs_init_impl_amd_avx+#define iq3xs_free_impl iq3xs_free_impl_amd_avx+#define ggml_validate_row_data ggml_validate_row_data_amd_avx+#include "ggml-quants.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-quants-amd-avx2.c b/llama.cpp/ggml-quants-amd-avx2.cnew file mode 100644index 00000000..3a12b6f4--- /dev/null+++ b/llama.cpp/ggml-quants-amd-avx2.c@@ -0,0 +1,100 @@+#ifdef __x86_64__+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_amd_avx2+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_amd_avx2+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_amd_avx2+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_amd_avx2+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_amd_avx2+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_amd_avx2+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_amd_avx2+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_amd_avx2+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_amd_avx2+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_amd_avx2+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_amd_avx2+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_amd_avx2+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_amd_avx2+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_amd_avx2+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_amd_avx2+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_amd_avx2+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_amd_avx2+#define quantize_row_q4_0 quantize_row_q4_0_amd_avx2+#define quantize_row_q4_1 quantize_row_q4_1_amd_avx2+#define quantize_row_q5_0 quantize_row_q5_0_amd_avx2+#define quantize_row_q5_1 quantize_row_q5_1_amd_avx2+#define quantize_row_q8_0 quantize_row_q8_0_amd_avx2+#define quantize_row_q8_1 quantize_row_q8_1_amd_avx2+#define quantize_row_q2_K quantize_row_q2_K_amd_avx2+#define quantize_row_q3_K quantize_row_q3_K_amd_avx2+#define quantize_row_q4_K quantize_row_q4_K_amd_avx2+#define quantize_row_q5_K quantize_row_q5_K_amd_avx2+#define quantize_row_q6_K quantize_row_q6_K_amd_avx2+#define quantize_row_q8_K quantize_row_q8_K_amd_avx2+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_amd_avx2+#define quantize_row_iq4_nl quantize_row_iq4_nl_amd_avx2+#define quantize_row_iq4_xs quantize_row_iq4_xs_amd_avx2+#define quantize_row_iq3_s quantize_row_iq3_s_amd_avx2+#define quantize_row_iq2_s quantize_row_iq2_s_amd_avx2+#define dequantize_row_q4_0 dequantize_row_q4_0_amd_avx2+#define dequantize_row_q4_1 dequantize_row_q4_1_amd_avx2+#define dequantize_row_q5_0 dequantize_row_q5_0_amd_avx2+#define dequantize_row_q5_1 dequantize_row_q5_1_amd_avx2+#define dequantize_row_q8_0 dequantize_row_q8_0_amd_avx2+#define dequantize_row_q2_K dequantize_row_q2_K_amd_avx2+#define dequantize_row_q3_K dequantize_row_q3_K_amd_avx2+#define dequantize_row_q4_K dequantize_row_q4_K_amd_avx2+#define dequantize_row_q5_K dequantize_row_q5_K_amd_avx2+#define dequantize_row_q6_K dequantize_row_q6_K_amd_avx2+#define dequantize_row_q8_K dequantize_row_q8_K_amd_avx2+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_amd_avx2+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_amd_avx2+#define dequantize_row_iq2_s dequantize_row_iq2_s_amd_avx2+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_amd_avx2+#define dequantize_row_iq1_s dequantize_row_iq1_s_amd_avx2+#define dequantize_row_iq1_m dequantize_row_iq1_m_amd_avx2+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_amd_avx2+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_amd_avx2+#define dequantize_row_iq3_s dequantize_row_iq3_s_amd_avx2+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_amd_avx2+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_amd_avx2+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_amd_avx2+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_amd_avx2+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_amd_avx2+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_amd_avx2+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_amd_avx2+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_amd_avx2+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_amd_avx2+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_amd_avx2+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_amd_avx2+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_amd_avx2+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_amd_avx2+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_amd_avx2+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_amd_avx2+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_amd_avx2+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_amd_avx2+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_amd_avx2+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_amd_avx2+#define quantize_iq2_xxs quantize_iq2_xxs_amd_avx2+#define quantize_iq2_xs quantize_iq2_xs_amd_avx2+#define quantize_iq2_s quantize_iq2_s_amd_avx2+#define quantize_iq3_xxs quantize_iq3_xxs_amd_avx2+#define quantize_iq1_s quantize_iq1_s_amd_avx2+#define quantize_iq1_m quantize_iq1_m_amd_avx2+#define quantize_iq4_nl quantize_iq4_nl_amd_avx2+#define quantize_iq4_xs quantize_iq4_xs_amd_avx2+#define quantize_iq3_s quantize_iq3_s_amd_avx2+#define quantize_q2_K quantize_q2_K_amd_avx2+#define quantize_q3_K quantize_q3_K_amd_avx2+#define quantize_q4_K quantize_q4_K_amd_avx2+#define quantize_q5_K quantize_q5_K_amd_avx2+#define quantize_q6_K quantize_q6_K_amd_avx2+#define quantize_q4_0 quantize_q4_0_amd_avx2+#define quantize_q4_1 quantize_q4_1_amd_avx2+#define quantize_q5_0 quantize_q5_0_amd_avx2+#define quantize_q5_1 quantize_q5_1_amd_avx2+#define quantize_q8_0 quantize_q8_0_amd_avx2+#define iq2xs_init_impl iq2xs_init_impl_amd_avx2+#define iq2xs_free_impl iq2xs_free_impl_amd_avx2+#define iq3xs_init_impl iq3xs_init_impl_amd_avx2+#define iq3xs_free_impl iq3xs_free_impl_amd_avx2+#define ggml_validate_row_data ggml_validate_row_data_amd_avx2+#include "ggml-quants.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-quants-amd-avx512.c b/llama.cpp/ggml-quants-amd-avx512.cnew file mode 100644index 00000000..590eff07--- /dev/null+++ b/llama.cpp/ggml-quants-amd-avx512.c@@ -0,0 +1,100 @@+#ifdef __x86_64__+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_amd_avx512+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_amd_avx512+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_amd_avx512+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_amd_avx512+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_amd_avx512+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_amd_avx512+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_amd_avx512+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_amd_avx512+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_amd_avx512+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_amd_avx512+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_amd_avx512+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_amd_avx512+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_amd_avx512+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_amd_avx512+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_amd_avx512+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_amd_avx512+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_amd_avx512+#define quantize_row_q4_0 quantize_row_q4_0_amd_avx512+#define quantize_row_q4_1 quantize_row_q4_1_amd_avx512+#define quantize_row_q5_0 quantize_row_q5_0_amd_avx512+#define quantize_row_q5_1 quantize_row_q5_1_amd_avx512+#define quantize_row_q8_0 quantize_row_q8_0_amd_avx512+#define quantize_row_q8_1 quantize_row_q8_1_amd_avx512+#define quantize_row_q2_K quantize_row_q2_K_amd_avx512+#define quantize_row_q3_K quantize_row_q3_K_amd_avx512+#define quantize_row_q4_K quantize_row_q4_K_amd_avx512+#define quantize_row_q5_K quantize_row_q5_K_amd_avx512+#define quantize_row_q6_K quantize_row_q6_K_amd_avx512+#define quantize_row_q8_K quantize_row_q8_K_amd_avx512+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_amd_avx512+#define quantize_row_iq4_nl quantize_row_iq4_nl_amd_avx512+#define quantize_row_iq4_xs quantize_row_iq4_xs_amd_avx512+#define quantize_row_iq3_s quantize_row_iq3_s_amd_avx512+#define quantize_row_iq2_s quantize_row_iq2_s_amd_avx512+#define dequantize_row_q4_0 dequantize_row_q4_0_amd_avx512+#define dequantize_row_q4_1 dequantize_row_q4_1_amd_avx512+#define dequantize_row_q5_0 dequantize_row_q5_0_amd_avx512+#define dequantize_row_q5_1 dequantize_row_q5_1_amd_avx512+#define dequantize_row_q8_0 dequantize_row_q8_0_amd_avx512+#define dequantize_row_q2_K dequantize_row_q2_K_amd_avx512+#define dequantize_row_q3_K dequantize_row_q3_K_amd_avx512+#define dequantize_row_q4_K dequantize_row_q4_K_amd_avx512+#define dequantize_row_q5_K dequantize_row_q5_K_amd_avx512+#define dequantize_row_q6_K dequantize_row_q6_K_amd_avx512+#define dequantize_row_q8_K dequantize_row_q8_K_amd_avx512+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_amd_avx512+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_amd_avx512+#define dequantize_row_iq2_s dequantize_row_iq2_s_amd_avx512+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_amd_avx512+#define dequantize_row_iq1_s dequantize_row_iq1_s_amd_avx512+#define dequantize_row_iq1_m dequantize_row_iq1_m_amd_avx512+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_amd_avx512+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_amd_avx512+#define dequantize_row_iq3_s dequantize_row_iq3_s_amd_avx512+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_amd_avx512+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_amd_avx512+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_amd_avx512+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_amd_avx512+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_amd_avx512+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_amd_avx512+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_amd_avx512+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_amd_avx512+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_amd_avx512+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_amd_avx512+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_amd_avx512+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_amd_avx512+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_amd_avx512+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_amd_avx512+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_amd_avx512+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_amd_avx512+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_amd_avx512+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_amd_avx512+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_amd_avx512+#define quantize_iq2_xxs quantize_iq2_xxs_amd_avx512+#define quantize_iq2_xs quantize_iq2_xs_amd_avx512+#define quantize_iq2_s quantize_iq2_s_amd_avx512+#define quantize_iq3_xxs quantize_iq3_xxs_amd_avx512+#define quantize_iq1_s quantize_iq1_s_amd_avx512+#define quantize_iq1_m quantize_iq1_m_amd_avx512+#define quantize_iq4_nl quantize_iq4_nl_amd_avx512+#define quantize_iq4_xs quantize_iq4_xs_amd_avx512+#define quantize_iq3_s quantize_iq3_s_amd_avx512+#define quantize_q2_K quantize_q2_K_amd_avx512+#define quantize_q3_K quantize_q3_K_amd_avx512+#define quantize_q4_K quantize_q4_K_amd_avx512+#define quantize_q5_K quantize_q5_K_amd_avx512+#define quantize_q6_K quantize_q6_K_amd_avx512+#define quantize_q4_0 quantize_q4_0_amd_avx512+#define quantize_q4_1 quantize_q4_1_amd_avx512+#define quantize_q5_0 quantize_q5_0_amd_avx512+#define quantize_q5_1 quantize_q5_1_amd_avx512+#define quantize_q8_0 quantize_q8_0_amd_avx512+#define iq2xs_init_impl iq2xs_init_impl_amd_avx512+#define iq2xs_free_impl iq2xs_free_impl_amd_avx512+#define iq3xs_init_impl iq3xs_init_impl_amd_avx512+#define iq3xs_free_impl iq3xs_free_impl_amd_avx512+#define ggml_validate_row_data ggml_validate_row_data_amd_avx512+#include "ggml-quants.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-quants-arm80.c b/llama.cpp/ggml-quants-arm80.cnew file mode 100644index 00000000..61783c01--- /dev/null+++ b/llama.cpp/ggml-quants-arm80.c@@ -0,0 +1,100 @@+#ifdef __aarch64__+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_arm80+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_arm80+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_arm80+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_arm80+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_arm80+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_arm80+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_arm80+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_arm80+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_arm80+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_arm80+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_arm80+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_arm80+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_arm80+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_arm80+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_arm80+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_arm80+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_arm80+#define quantize_row_q4_0 quantize_row_q4_0_arm80+#define quantize_row_q4_1 quantize_row_q4_1_arm80+#define quantize_row_q5_0 quantize_row_q5_0_arm80+#define quantize_row_q5_1 quantize_row_q5_1_arm80+#define quantize_row_q8_0 quantize_row_q8_0_arm80+#define quantize_row_q8_1 quantize_row_q8_1_arm80+#define quantize_row_q2_K quantize_row_q2_K_arm80+#define quantize_row_q3_K quantize_row_q3_K_arm80+#define quantize_row_q4_K quantize_row_q4_K_arm80+#define quantize_row_q5_K quantize_row_q5_K_arm80+#define quantize_row_q6_K quantize_row_q6_K_arm80+#define quantize_row_q8_K quantize_row_q8_K_arm80+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_arm80+#define quantize_row_iq4_nl quantize_row_iq4_nl_arm80+#define quantize_row_iq4_xs quantize_row_iq4_xs_arm80+#define quantize_row_iq3_s quantize_row_iq3_s_arm80+#define quantize_row_iq2_s quantize_row_iq2_s_arm80+#define dequantize_row_q4_0 dequantize_row_q4_0_arm80+#define dequantize_row_q4_1 dequantize_row_q4_1_arm80+#define dequantize_row_q5_0 dequantize_row_q5_0_arm80+#define dequantize_row_q5_1 dequantize_row_q5_1_arm80+#define dequantize_row_q8_0 dequantize_row_q8_0_arm80+#define dequantize_row_q2_K dequantize_row_q2_K_arm80+#define dequantize_row_q3_K dequantize_row_q3_K_arm80+#define dequantize_row_q4_K dequantize_row_q4_K_arm80+#define dequantize_row_q5_K dequantize_row_q5_K_arm80+#define dequantize_row_q6_K dequantize_row_q6_K_arm80+#define dequantize_row_q8_K dequantize_row_q8_K_arm80+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_arm80+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_arm80+#define dequantize_row_iq2_s dequantize_row_iq2_s_arm80+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_arm80+#define dequantize_row_iq1_s dequantize_row_iq1_s_arm80+#define dequantize_row_iq1_m dequantize_row_iq1_m_arm80+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_arm80+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_arm80+#define dequantize_row_iq3_s dequantize_row_iq3_s_arm80+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_arm80+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_arm80+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_arm80+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_arm80+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_arm80+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_arm80+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_arm80+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_arm80+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_arm80+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_arm80+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_arm80+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_arm80+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_arm80+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_arm80+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_arm80+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_arm80+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_arm80+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_arm80+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_arm80+#define quantize_iq2_xxs quantize_iq2_xxs_arm80+#define quantize_iq2_xs quantize_iq2_xs_arm80+#define quantize_iq2_s quantize_iq2_s_arm80+#define quantize_iq3_xxs quantize_iq3_xxs_arm80+#define quantize_iq1_s quantize_iq1_s_arm80+#define quantize_iq1_m quantize_iq1_m_arm80+#define quantize_iq4_nl quantize_iq4_nl_arm80+#define quantize_iq4_xs quantize_iq4_xs_arm80+#define quantize_iq3_s quantize_iq3_s_arm80+#define quantize_q2_K quantize_q2_K_arm80+#define quantize_q3_K quantize_q3_K_arm80+#define quantize_q4_K quantize_q4_K_arm80+#define quantize_q5_K quantize_q5_K_arm80+#define quantize_q6_K quantize_q6_K_arm80+#define quantize_q4_0 quantize_q4_0_arm80+#define quantize_q4_1 quantize_q4_1_arm80+#define quantize_q5_0 quantize_q5_0_arm80+#define quantize_q5_1 quantize_q5_1_arm80+#define quantize_q8_0 quantize_q8_0_arm80+#define iq2xs_init_impl iq2xs_init_impl_arm80+#define iq2xs_free_impl iq2xs_free_impl_arm80+#define iq3xs_init_impl iq3xs_init_impl_arm80+#define iq3xs_free_impl iq3xs_free_impl_arm80+#define ggml_validate_row_data ggml_validate_row_data_arm80+#include "ggml-quants.inc"+#endif // __aarch64__diff --git a/llama.cpp/ggml-quants.cpp b/llama.cpp/ggml-quants.cppnew file mode 100644index 00000000..797d0990--- /dev/null+++ b/llama.cpp/ggml-quants.cpp@@ -0,0 +1,1381 @@+#include <cosmo.h>+#include <sys/auxv.h>+#include <libc/sysv/consts/hwcap.h>+#include "ggml-quants.h"++extern "C" void quantize_row_q4_0_reference_amd_avx512(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_0_reference_amd_avx2(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_0_reference_amd_avx(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_0_reference_arm80(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q4_1_reference_amd_avx512(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_1_reference_amd_avx2(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_1_reference_amd_avx(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_1_reference_arm80(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q5_0_reference_amd_avx512(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_0_reference_amd_avx2(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_0_reference_amd_avx(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_0_reference_arm80(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q5_1_reference_amd_avx512(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_1_reference_amd_avx2(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_1_reference_amd_avx(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_1_reference_arm80(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q8_0_reference_amd_avx512(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_0_reference_amd_avx2(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_0_reference_amd_avx(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_0_reference_arm80(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q8_1_reference_amd_avx512(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_1_reference_amd_avx2(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_1_reference_amd_avx(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_1_reference_arm80(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q2_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q2_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q2_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q2_K_reference_arm80(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q3_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q3_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q3_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q3_K_reference_arm80(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q4_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_K_reference_arm80(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q5_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_K_reference_arm80(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q6_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q6_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q6_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q6_K_reference_arm80(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q8_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_K_reference_arm80(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq3_xxs_reference_amd_avx512(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_xxs_reference_amd_avx2(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_xxs_reference_amd_avx(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_xxs_reference_arm80(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq4_nl_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_nl_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_nl_reference_amd_avx (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_nl_reference_arm80 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq4_xs_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_xs_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_xs_reference_amd_avx (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_xs_reference_arm80 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq3_s_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_s_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_s_reference_amd_avx (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_s_reference_arm80 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq2_s_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq2_s_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq2_s_reference_amd_avx (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq2_s_reference_arm80 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q4_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q4_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q5_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q5_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q8_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q8_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q2_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q2_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q2_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q2_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q3_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q3_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q3_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q3_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q4_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q4_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q5_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q5_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q6_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q6_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q6_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q6_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_q8_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_q8_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq3_xxs_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_xxs_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_xxs_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_xxs_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq4_nl_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_nl_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_nl_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_nl_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq4_xs_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_xs_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_xs_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq4_xs_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq3_s_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_s_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_s_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq3_s_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void quantize_row_iq2_s_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq2_s_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq2_s_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);+extern "C" void quantize_row_iq2_s_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q4_0_amd_avx512(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_0_amd_avx2(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_0_amd_avx(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_0_arm80(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q4_1_amd_avx512(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_1_amd_avx2(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_1_amd_avx(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_1_arm80(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q5_0_amd_avx512(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_0_amd_avx2(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_0_amd_avx(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_0_arm80(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q5_1_amd_avx512(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_1_amd_avx2(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_1_amd_avx(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_1_arm80(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q8_0_amd_avx512(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q8_0_amd_avx2(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q8_0_amd_avx(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q8_0_arm80(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q2_K_amd_avx512(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q2_K_amd_avx2(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q2_K_amd_avx(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q2_K_arm80(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q3_K_amd_avx512(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q3_K_amd_avx2(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q3_K_amd_avx(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q3_K_arm80(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q4_K_amd_avx512(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_K_amd_avx2(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_K_amd_avx(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q4_K_arm80(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q5_K_amd_avx512(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_K_amd_avx2(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_K_amd_avx(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q5_K_arm80(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q6_K_amd_avx512(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q6_K_amd_avx2(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q6_K_amd_avx(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q6_K_arm80(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_q8_K_amd_avx512(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q8_K_amd_avx2(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q8_K_amd_avx(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_q8_K_arm80(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq2_xxs_amd_avx512(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_xxs_amd_avx2(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_xxs_amd_avx(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_xxs_arm80(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq2_xs_amd_avx512 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_xs_amd_avx2 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_xs_amd_avx (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_xs_arm80 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq2_s_amd_avx512 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_s_amd_avx2 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_s_amd_avx (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq2_s_arm80 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq3_xxs_amd_avx512(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq3_xxs_amd_avx2(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq3_xxs_amd_avx(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq3_xxs_arm80(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq1_s_amd_avx512 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq1_s_amd_avx2 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq1_s_amd_avx (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq1_s_arm80 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq1_m_amd_avx512 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq1_m_amd_avx2 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq1_m_amd_avx (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq1_m_arm80 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq4_nl_amd_avx512 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq4_nl_amd_avx2 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq4_nl_amd_avx (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq4_nl_arm80 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq4_xs_amd_avx512 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq4_xs_amd_avx2 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq4_xs_amd_avx (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq4_xs_arm80 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void dequantize_row_iq3_s_amd_avx512 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq3_s_amd_avx2 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq3_s_amd_avx (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+extern "C" void dequantize_row_iq3_s_arm80 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);++extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_1_q8_1_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_1_q8_1_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q8_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q2_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q3_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q4_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q5_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_q6_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_xs_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq2_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq1_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq1_m_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq4_nl_q8_0_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq4_xs_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+extern "C" void ggml_vec_dot_iq3_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++extern "C" size_t quantize_iq2_xxs_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_xxs_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_xxs_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_xxs_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq2_xs_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_xs_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_xs_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_xs_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq2_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq2_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq3_xxs_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq3_xxs_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq3_xxs_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq3_xxs_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq1_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq1_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq1_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq1_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq1_m_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq1_m_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq1_m_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq1_m_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq4_nl_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq4_nl_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq4_nl_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq4_nl_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq4_xs_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq4_xs_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq4_xs_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq4_xs_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_iq3_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq3_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq3_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_iq3_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q2_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q2_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q2_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q2_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q3_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q3_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q3_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q3_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q4_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q5_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q6_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q6_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q6_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q6_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q4_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q4_1_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_1_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_1_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q4_1_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q5_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q5_1_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_1_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_1_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q5_1_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" size_t quantize_q8_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q8_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q8_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);+extern "C" size_t quantize_q8_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);++extern "C" void iq2xs_init_impl_amd_avx512(enum ggml_type type);+extern "C" void iq2xs_init_impl_amd_avx2(enum ggml_type type);+extern "C" void iq2xs_init_impl_amd_avx(enum ggml_type type);+extern "C" void iq2xs_init_impl_arm80(enum ggml_type type);++extern "C" void iq2xs_free_impl_amd_avx512(enum ggml_type type);+extern "C" void iq2xs_free_impl_amd_avx2(enum ggml_type type);+extern "C" void iq2xs_free_impl_amd_avx(enum ggml_type type);+extern "C" void iq2xs_free_impl_arm80(enum ggml_type type);++extern "C" void iq3xs_init_impl_amd_avx512(int grid_size);+extern "C" void iq3xs_init_impl_amd_avx2(int grid_size);+extern "C" void iq3xs_init_impl_amd_avx(int grid_size);+extern "C" void iq3xs_init_impl_arm80(int grid_size);++extern "C" void iq3xs_free_impl_amd_avx512(int grid_size);+extern "C" void iq3xs_free_impl_amd_avx2(int grid_size);+extern "C" void iq3xs_free_impl_amd_avx(int grid_size);+extern "C" void iq3xs_free_impl_arm80(int grid_size);++extern "C" bool ggml_validate_row_data_amd_avx512(enum ggml_type type, const void * data, size_t nbytes);+extern "C" bool ggml_validate_row_data_amd_avx2(enum ggml_type type, const void * data, size_t nbytes);+extern "C" bool ggml_validate_row_data_amd_avx(enum ggml_type type, const void * data, size_t nbytes);+extern "C" bool ggml_validate_row_data_arm80(enum ggml_type type, const void * data, size_t nbytes);++static const struct QuantFuncs {+ typeof(quantize_row_q4_0_reference) *ptr_quantize_row_q4_0_reference;+ typeof(quantize_row_q4_1_reference) *ptr_quantize_row_q4_1_reference;+ typeof(quantize_row_q5_0_reference) *ptr_quantize_row_q5_0_reference;+ typeof(quantize_row_q5_1_reference) *ptr_quantize_row_q5_1_reference;+ typeof(quantize_row_q8_0_reference) *ptr_quantize_row_q8_0_reference;+ typeof(quantize_row_q8_1_reference) *ptr_quantize_row_q8_1_reference;+ typeof(quantize_row_q2_K_reference) *ptr_quantize_row_q2_K_reference;+ typeof(quantize_row_q3_K_reference) *ptr_quantize_row_q3_K_reference;+ typeof(quantize_row_q4_K_reference) *ptr_quantize_row_q4_K_reference;+ typeof(quantize_row_q5_K_reference) *ptr_quantize_row_q5_K_reference;+ typeof(quantize_row_q6_K_reference) *ptr_quantize_row_q6_K_reference;+ typeof(quantize_row_q8_K_reference) *ptr_quantize_row_q8_K_reference;+ typeof(quantize_row_iq3_xxs_reference) *ptr_quantize_row_iq3_xxs_reference;+ typeof(quantize_row_iq4_nl_reference) *ptr_quantize_row_iq4_nl_reference;+ typeof(quantize_row_iq4_xs_reference) *ptr_quantize_row_iq4_xs_reference;+ typeof(quantize_row_iq3_s_reference) *ptr_quantize_row_iq3_s_reference;+ typeof(quantize_row_iq2_s_reference) *ptr_quantize_row_iq2_s_reference;+ typeof(quantize_row_q4_0) *ptr_quantize_row_q4_0;+ typeof(quantize_row_q4_1) *ptr_quantize_row_q4_1;+ typeof(quantize_row_q5_0) *ptr_quantize_row_q5_0;+ typeof(quantize_row_q5_1) *ptr_quantize_row_q5_1;+ typeof(quantize_row_q8_0) *ptr_quantize_row_q8_0;+ typeof(quantize_row_q8_1) *ptr_quantize_row_q8_1;+ typeof(quantize_row_q2_K) *ptr_quantize_row_q2_K;+ typeof(quantize_row_q3_K) *ptr_quantize_row_q3_K;+ typeof(quantize_row_q4_K) *ptr_quantize_row_q4_K;+ typeof(quantize_row_q5_K) *ptr_quantize_row_q5_K;+ typeof(quantize_row_q6_K) *ptr_quantize_row_q6_K;+ typeof(quantize_row_q8_K) *ptr_quantize_row_q8_K;+ typeof(quantize_row_iq3_xxs) *ptr_quantize_row_iq3_xxs;+ typeof(quantize_row_iq4_nl) *ptr_quantize_row_iq4_nl;+ typeof(quantize_row_iq4_xs) *ptr_quantize_row_iq4_xs;+ typeof(quantize_row_iq3_s) *ptr_quantize_row_iq3_s;+ typeof(quantize_row_iq2_s) *ptr_quantize_row_iq2_s;+ typeof(dequantize_row_q4_0) *ptr_dequantize_row_q4_0;+ typeof(dequantize_row_q4_1) *ptr_dequantize_row_q4_1;+ typeof(dequantize_row_q5_0) *ptr_dequantize_row_q5_0;+ typeof(dequantize_row_q5_1) *ptr_dequantize_row_q5_1;+ typeof(dequantize_row_q8_0) *ptr_dequantize_row_q8_0;+ typeof(dequantize_row_q2_K) *ptr_dequantize_row_q2_K;+ typeof(dequantize_row_q3_K) *ptr_dequantize_row_q3_K;+ typeof(dequantize_row_q4_K) *ptr_dequantize_row_q4_K;+ typeof(dequantize_row_q5_K) *ptr_dequantize_row_q5_K;+ typeof(dequantize_row_q6_K) *ptr_dequantize_row_q6_K;+ typeof(dequantize_row_q8_K) *ptr_dequantize_row_q8_K;+ typeof(dequantize_row_iq2_xxs) *ptr_dequantize_row_iq2_xxs;+ typeof(dequantize_row_iq2_xs) *ptr_dequantize_row_iq2_xs;+ typeof(dequantize_row_iq2_s) *ptr_dequantize_row_iq2_s;+ typeof(dequantize_row_iq3_xxs) *ptr_dequantize_row_iq3_xxs;+ typeof(dequantize_row_iq1_s) *ptr_dequantize_row_iq1_s;+ typeof(dequantize_row_iq1_m) *ptr_dequantize_row_iq1_m;+ typeof(dequantize_row_iq4_nl) *ptr_dequantize_row_iq4_nl;+ typeof(dequantize_row_iq4_xs) *ptr_dequantize_row_iq4_xs;+ typeof(dequantize_row_iq3_s) *ptr_dequantize_row_iq3_s;+ typeof(ggml_vec_dot_q4_0_q8_0) *ptr_ggml_vec_dot_q4_0_q8_0;+ typeof(ggml_vec_dot_q4_1_q8_1) *ptr_ggml_vec_dot_q4_1_q8_1;+ typeof(ggml_vec_dot_q5_0_q8_0) *ptr_ggml_vec_dot_q5_0_q8_0;+ typeof(ggml_vec_dot_q5_1_q8_1) *ptr_ggml_vec_dot_q5_1_q8_1;+ typeof(ggml_vec_dot_q8_0_q8_0) *ptr_ggml_vec_dot_q8_0_q8_0;+ typeof(ggml_vec_dot_q2_K_q8_K) *ptr_ggml_vec_dot_q2_K_q8_K;+ typeof(ggml_vec_dot_q3_K_q8_K) *ptr_ggml_vec_dot_q3_K_q8_K;+ typeof(ggml_vec_dot_q4_K_q8_K) *ptr_ggml_vec_dot_q4_K_q8_K;+ typeof(ggml_vec_dot_q5_K_q8_K) *ptr_ggml_vec_dot_q5_K_q8_K;+ typeof(ggml_vec_dot_q6_K_q8_K) *ptr_ggml_vec_dot_q6_K_q8_K;+ typeof(ggml_vec_dot_iq2_xxs_q8_K) *ptr_ggml_vec_dot_iq2_xxs_q8_K;+ typeof(ggml_vec_dot_iq2_xs_q8_K) *ptr_ggml_vec_dot_iq2_xs_q8_K;+ typeof(ggml_vec_dot_iq2_s_q8_K) *ptr_ggml_vec_dot_iq2_s_q8_K;+ typeof(ggml_vec_dot_iq3_xxs_q8_K) *ptr_ggml_vec_dot_iq3_xxs_q8_K;+ typeof(ggml_vec_dot_iq1_s_q8_K) *ptr_ggml_vec_dot_iq1_s_q8_K;+ typeof(ggml_vec_dot_iq1_m_q8_K) *ptr_ggml_vec_dot_iq1_m_q8_K;+ typeof(ggml_vec_dot_iq4_nl_q8_0) *ptr_ggml_vec_dot_iq4_nl_q8_0;+ typeof(ggml_vec_dot_iq4_xs_q8_K) *ptr_ggml_vec_dot_iq4_xs_q8_K;+ typeof(ggml_vec_dot_iq3_s_q8_K) *ptr_ggml_vec_dot_iq3_s_q8_K;+ typeof(quantize_iq2_xxs) *ptr_quantize_iq2_xxs;+ typeof(quantize_iq2_xs) *ptr_quantize_iq2_xs;+ typeof(quantize_iq2_s) *ptr_quantize_iq2_s;+ typeof(quantize_iq3_xxs) *ptr_quantize_iq3_xxs;+ typeof(quantize_iq1_s) *ptr_quantize_iq1_s;+ typeof(quantize_iq1_m) *ptr_quantize_iq1_m;+ typeof(quantize_iq4_nl) *ptr_quantize_iq4_nl;+ typeof(quantize_iq4_xs) *ptr_quantize_iq4_xs;+ typeof(quantize_iq3_s) *ptr_quantize_iq3_s;+ typeof(quantize_q2_K) *ptr_quantize_q2_K;+ typeof(quantize_q3_K) *ptr_quantize_q3_K;+ typeof(quantize_q4_K) *ptr_quantize_q4_K;+ typeof(quantize_q5_K) *ptr_quantize_q5_K;+ typeof(quantize_q6_K) *ptr_quantize_q6_K;+ typeof(quantize_q4_0) *ptr_quantize_q4_0;+ typeof(quantize_q4_1) *ptr_quantize_q4_1;+ typeof(quantize_q5_0) *ptr_quantize_q5_0;+ typeof(quantize_q5_1) *ptr_quantize_q5_1;+ typeof(quantize_q8_0) *ptr_quantize_q8_0;+ typeof(iq2xs_init_impl) *ptr_iq2xs_init_impl;+ typeof(iq2xs_free_impl) *ptr_iq2xs_free_impl;+ typeof(iq3xs_init_impl) *ptr_iq3xs_init_impl;+ typeof(iq3xs_free_impl) *ptr_iq3xs_free_impl;+ typeof(ggml_validate_row_data) *ptr_ggml_validate_row_data;++ QuantFuncs() {+#ifdef __x86_64__+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F)) {+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_amd_avx512;+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_amd_avx512;+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_amd_avx512;+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_amd_avx512;+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_amd_avx512;+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_amd_avx512;+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_amd_avx512;+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_amd_avx512;+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_amd_avx512;+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_amd_avx512;+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_amd_avx512;+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_amd_avx512;+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_amd_avx512;+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_amd_avx512;+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_amd_avx512;+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_amd_avx512;+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_amd_avx512;+ ptr_quantize_row_q4_0 = quantize_row_q4_0_amd_avx512;+ ptr_quantize_row_q4_1 = quantize_row_q4_1_amd_avx512;+ ptr_quantize_row_q5_0 = quantize_row_q5_0_amd_avx512;+ ptr_quantize_row_q5_1 = quantize_row_q5_1_amd_avx512;+ ptr_quantize_row_q8_0 = quantize_row_q8_0_amd_avx512;+ ptr_quantize_row_q8_1 = quantize_row_q8_1_amd_avx512;+ ptr_quantize_row_q2_K = quantize_row_q2_K_amd_avx512;+ ptr_quantize_row_q3_K = quantize_row_q3_K_amd_avx512;+ ptr_quantize_row_q4_K = quantize_row_q4_K_amd_avx512;+ ptr_quantize_row_q5_K = quantize_row_q5_K_amd_avx512;+ ptr_quantize_row_q6_K = quantize_row_q6_K_amd_avx512;+ ptr_quantize_row_q8_K = quantize_row_q8_K_amd_avx512;+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_amd_avx512;+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_amd_avx512;+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_amd_avx512;+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_amd_avx512;+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_amd_avx512;+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_amd_avx512;+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_amd_avx512;+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_amd_avx512;+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_amd_avx512;+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_amd_avx512;+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_amd_avx512;+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_amd_avx512;+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_amd_avx512;+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_amd_avx512;+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_amd_avx512;+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_amd_avx512;+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_amd_avx512;+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_amd_avx512;+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_amd_avx512;+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_amd_avx512;+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_amd_avx512;+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_amd_avx512;+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_amd_avx512;+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_amd_avx512;+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_amd_avx512;+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_amd_avx512;+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_amd_avx512;+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_amd_avx512;+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_amd_avx512;+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_amd_avx512;+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_amd_avx512;+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_amd_avx512;+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_amd_avx512;+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_amd_avx512;+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_amd_avx512;+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_amd_avx512;+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_amd_avx512;+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_amd_avx512;+ ptr_quantize_iq2_xs = quantize_iq2_xs_amd_avx512;+ ptr_quantize_iq2_s = quantize_iq2_s_amd_avx512;+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_amd_avx512;+ ptr_quantize_iq1_s = quantize_iq1_s_amd_avx512;+ ptr_quantize_iq1_m = quantize_iq1_m_amd_avx512;+ ptr_quantize_iq4_nl = quantize_iq4_nl_amd_avx512;+ ptr_quantize_iq4_xs = quantize_iq4_xs_amd_avx512;+ ptr_quantize_iq3_s = quantize_iq3_s_amd_avx512;+ ptr_quantize_q2_K = quantize_q2_K_amd_avx512;+ ptr_quantize_q3_K = quantize_q3_K_amd_avx512;+ ptr_quantize_q4_K = quantize_q4_K_amd_avx512;+ ptr_quantize_q5_K = quantize_q5_K_amd_avx512;+ ptr_quantize_q6_K = quantize_q6_K_amd_avx512;+ ptr_quantize_q4_0 = quantize_q4_0_amd_avx512;+ ptr_quantize_q4_1 = quantize_q4_1_amd_avx512;+ ptr_quantize_q5_0 = quantize_q5_0_amd_avx512;+ ptr_quantize_q5_1 = quantize_q5_1_amd_avx512;+ ptr_quantize_q8_0 = quantize_q8_0_amd_avx512;+ ptr_iq2xs_init_impl = iq2xs_init_impl_amd_avx512;+ ptr_iq2xs_free_impl = iq2xs_free_impl_amd_avx512;+ ptr_iq3xs_init_impl = iq3xs_init_impl_amd_avx512;+ ptr_iq3xs_free_impl = iq3xs_free_impl_amd_avx512;+ ptr_ggml_validate_row_data = ggml_validate_row_data_amd_avx512;+ return;+ }+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2)) {+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_amd_avx2;+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_amd_avx2;+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_amd_avx2;+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_amd_avx2;+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_amd_avx2;+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_amd_avx2;+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_amd_avx2;+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_amd_avx2;+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_amd_avx2;+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_amd_avx2;+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_amd_avx2;+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_amd_avx2;+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_amd_avx2;+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_amd_avx2;+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_amd_avx2;+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_amd_avx2;+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_amd_avx2;+ ptr_quantize_row_q4_0 = quantize_row_q4_0_amd_avx2;+ ptr_quantize_row_q4_1 = quantize_row_q4_1_amd_avx2;+ ptr_quantize_row_q5_0 = quantize_row_q5_0_amd_avx2;+ ptr_quantize_row_q5_1 = quantize_row_q5_1_amd_avx2;+ ptr_quantize_row_q8_0 = quantize_row_q8_0_amd_avx2;+ ptr_quantize_row_q8_1 = quantize_row_q8_1_amd_avx2;+ ptr_quantize_row_q2_K = quantize_row_q2_K_amd_avx2;+ ptr_quantize_row_q3_K = quantize_row_q3_K_amd_avx2;+ ptr_quantize_row_q4_K = quantize_row_q4_K_amd_avx2;+ ptr_quantize_row_q5_K = quantize_row_q5_K_amd_avx2;+ ptr_quantize_row_q6_K = quantize_row_q6_K_amd_avx2;+ ptr_quantize_row_q8_K = quantize_row_q8_K_amd_avx2;+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_amd_avx2;+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_amd_avx2;+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_amd_avx2;+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_amd_avx2;+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_amd_avx2;+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_amd_avx2;+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_amd_avx2;+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_amd_avx2;+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_amd_avx2;+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_amd_avx2;+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_amd_avx2;+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_amd_avx2;+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_amd_avx2;+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_amd_avx2;+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_amd_avx2;+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_amd_avx2;+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_amd_avx2;+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_amd_avx2;+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_amd_avx2;+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_amd_avx2;+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_amd_avx2;+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_amd_avx2;+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_amd_avx2;+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_amd_avx2;+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_amd_avx2;+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_amd_avx2;+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_amd_avx2;+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_amd_avx2;+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_amd_avx2;+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_amd_avx2;+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_amd_avx2;+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_amd_avx2;+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_amd_avx2;+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_amd_avx2;+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_amd_avx2;+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_amd_avx2;+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_amd_avx2;+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_amd_avx2;+ ptr_quantize_iq2_xs = quantize_iq2_xs_amd_avx2;+ ptr_quantize_iq2_s = quantize_iq2_s_amd_avx2;+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_amd_avx2;+ ptr_quantize_iq1_s = quantize_iq1_s_amd_avx2;+ ptr_quantize_iq1_m = quantize_iq1_m_amd_avx2;+ ptr_quantize_iq4_nl = quantize_iq4_nl_amd_avx2;+ ptr_quantize_iq4_xs = quantize_iq4_xs_amd_avx2;+ ptr_quantize_iq3_s = quantize_iq3_s_amd_avx2;+ ptr_quantize_q2_K = quantize_q2_K_amd_avx2;+ ptr_quantize_q3_K = quantize_q3_K_amd_avx2;+ ptr_quantize_q4_K = quantize_q4_K_amd_avx2;+ ptr_quantize_q5_K = quantize_q5_K_amd_avx2;+ ptr_quantize_q6_K = quantize_q6_K_amd_avx2;+ ptr_quantize_q4_0 = quantize_q4_0_amd_avx2;+ ptr_quantize_q4_1 = quantize_q4_1_amd_avx2;+ ptr_quantize_q5_0 = quantize_q5_0_amd_avx2;+ ptr_quantize_q5_1 = quantize_q5_1_amd_avx2;+ ptr_quantize_q8_0 = quantize_q8_0_amd_avx2;+ ptr_iq2xs_init_impl = iq2xs_init_impl_amd_avx2;+ ptr_iq2xs_free_impl = iq2xs_free_impl_amd_avx2;+ ptr_iq3xs_init_impl = iq3xs_init_impl_amd_avx2;+ ptr_iq3xs_free_impl = iq3xs_free_impl_amd_avx2;+ ptr_ggml_validate_row_data = ggml_validate_row_data_amd_avx2;+ return;+ }+ if (1) {+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_amd_avx;+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_amd_avx;+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_amd_avx;+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_amd_avx;+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_amd_avx;+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_amd_avx;+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_amd_avx;+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_amd_avx;+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_amd_avx;+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_amd_avx;+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_amd_avx;+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_amd_avx;+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_amd_avx;+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_amd_avx;+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_amd_avx;+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_amd_avx;+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_amd_avx;+ ptr_quantize_row_q4_0 = quantize_row_q4_0_amd_avx;+ ptr_quantize_row_q4_1 = quantize_row_q4_1_amd_avx;+ ptr_quantize_row_q5_0 = quantize_row_q5_0_amd_avx;+ ptr_quantize_row_q5_1 = quantize_row_q5_1_amd_avx;+ ptr_quantize_row_q8_0 = quantize_row_q8_0_amd_avx;+ ptr_quantize_row_q8_1 = quantize_row_q8_1_amd_avx;+ ptr_quantize_row_q2_K = quantize_row_q2_K_amd_avx;+ ptr_quantize_row_q3_K = quantize_row_q3_K_amd_avx;+ ptr_quantize_row_q4_K = quantize_row_q4_K_amd_avx;+ ptr_quantize_row_q5_K = quantize_row_q5_K_amd_avx;+ ptr_quantize_row_q6_K = quantize_row_q6_K_amd_avx;+ ptr_quantize_row_q8_K = quantize_row_q8_K_amd_avx;+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_amd_avx;+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_amd_avx;+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_amd_avx;+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_amd_avx;+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_amd_avx;+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_amd_avx;+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_amd_avx;+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_amd_avx;+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_amd_avx;+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_amd_avx;+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_amd_avx;+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_amd_avx;+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_amd_avx;+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_amd_avx;+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_amd_avx;+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_amd_avx;+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_amd_avx;+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_amd_avx;+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_amd_avx;+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_amd_avx;+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_amd_avx;+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_amd_avx;+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_amd_avx;+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_amd_avx;+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_amd_avx;+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_amd_avx;+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_amd_avx;+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_amd_avx;+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_amd_avx;+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_amd_avx;+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_amd_avx;+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_amd_avx;+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_amd_avx;+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_amd_avx;+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_amd_avx;+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_amd_avx;+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_amd_avx;+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_amd_avx;+ ptr_quantize_iq2_xs = quantize_iq2_xs_amd_avx;+ ptr_quantize_iq2_s = quantize_iq2_s_amd_avx;+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_amd_avx;+ ptr_quantize_iq1_s = quantize_iq1_s_amd_avx;+ ptr_quantize_iq1_m = quantize_iq1_m_amd_avx;+ ptr_quantize_iq4_nl = quantize_iq4_nl_amd_avx;+ ptr_quantize_iq4_xs = quantize_iq4_xs_amd_avx;+ ptr_quantize_iq3_s = quantize_iq3_s_amd_avx;+ ptr_quantize_q2_K = quantize_q2_K_amd_avx;+ ptr_quantize_q3_K = quantize_q3_K_amd_avx;+ ptr_quantize_q4_K = quantize_q4_K_amd_avx;+ ptr_quantize_q5_K = quantize_q5_K_amd_avx;+ ptr_quantize_q6_K = quantize_q6_K_amd_avx;+ ptr_quantize_q4_0 = quantize_q4_0_amd_avx;+ ptr_quantize_q4_1 = quantize_q4_1_amd_avx;+ ptr_quantize_q5_0 = quantize_q5_0_amd_avx;+ ptr_quantize_q5_1 = quantize_q5_1_amd_avx;+ ptr_quantize_q8_0 = quantize_q8_0_amd_avx;+ ptr_iq2xs_init_impl = iq2xs_init_impl_amd_avx;+ ptr_iq2xs_free_impl = iq2xs_free_impl_amd_avx;+ ptr_iq3xs_init_impl = iq3xs_init_impl_amd_avx;+ ptr_iq3xs_free_impl = iq3xs_free_impl_amd_avx;+ ptr_ggml_validate_row_data = ggml_validate_row_data_amd_avx;+ return;+ }+#else+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_arm80;+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_arm80;+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_arm80;+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_arm80;+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_arm80;+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_arm80;+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_arm80;+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_arm80;+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_arm80;+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_arm80;+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_arm80;+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_arm80;+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_arm80;+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_arm80;+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_arm80;+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_arm80;+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_arm80;+ ptr_quantize_row_q4_0 = quantize_row_q4_0_arm80;+ ptr_quantize_row_q4_1 = quantize_row_q4_1_arm80;+ ptr_quantize_row_q5_0 = quantize_row_q5_0_arm80;+ ptr_quantize_row_q5_1 = quantize_row_q5_1_arm80;+ ptr_quantize_row_q8_0 = quantize_row_q8_0_arm80;+ ptr_quantize_row_q8_1 = quantize_row_q8_1_arm80;+ ptr_quantize_row_q2_K = quantize_row_q2_K_arm80;+ ptr_quantize_row_q3_K = quantize_row_q3_K_arm80;+ ptr_quantize_row_q4_K = quantize_row_q4_K_arm80;+ ptr_quantize_row_q5_K = quantize_row_q5_K_arm80;+ ptr_quantize_row_q6_K = quantize_row_q6_K_arm80;+ ptr_quantize_row_q8_K = quantize_row_q8_K_arm80;+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_arm80;+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_arm80;+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_arm80;+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_arm80;+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_arm80;+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_arm80;+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_arm80;+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_arm80;+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_arm80;+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_arm80;+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_arm80;+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_arm80;+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_arm80;+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_arm80;+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_arm80;+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_arm80;+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_arm80;+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_arm80;+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_arm80;+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_arm80;+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_arm80;+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_arm80;+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_arm80;+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_arm80;+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_arm80;+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_arm80;+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_arm80;+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_arm80;+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_arm80;+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_arm80;+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_arm80;+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_arm80;+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_arm80;+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_arm80;+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_arm80;+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_arm80;+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_arm80;+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_arm80;+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_arm80;+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_arm80;+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_arm80;+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_arm80;+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_arm80;+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_arm80;+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_arm80;+ ptr_quantize_iq2_xs = quantize_iq2_xs_arm80;+ ptr_quantize_iq2_s = quantize_iq2_s_arm80;+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_arm80;+ ptr_quantize_iq1_s = quantize_iq1_s_arm80;+ ptr_quantize_iq1_m = quantize_iq1_m_arm80;+ ptr_quantize_iq4_nl = quantize_iq4_nl_arm80;+ ptr_quantize_iq4_xs = quantize_iq4_xs_arm80;+ ptr_quantize_iq3_s = quantize_iq3_s_arm80;+ ptr_quantize_q2_K = quantize_q2_K_arm80;+ ptr_quantize_q3_K = quantize_q3_K_arm80;+ ptr_quantize_q4_K = quantize_q4_K_arm80;+ ptr_quantize_q5_K = quantize_q5_K_arm80;+ ptr_quantize_q6_K = quantize_q6_K_arm80;+ ptr_quantize_q4_0 = quantize_q4_0_arm80;+ ptr_quantize_q4_1 = quantize_q4_1_arm80;+ ptr_quantize_q5_0 = quantize_q5_0_arm80;+ ptr_quantize_q5_1 = quantize_q5_1_arm80;+ ptr_quantize_q8_0 = quantize_q8_0_arm80;+ ptr_iq2xs_init_impl = iq2xs_init_impl_arm80;+ ptr_iq2xs_free_impl = iq2xs_free_impl_arm80;+ ptr_iq3xs_init_impl = iq3xs_init_impl_arm80;+ ptr_iq3xs_free_impl = iq3xs_free_impl_arm80;+ ptr_ggml_validate_row_data = ggml_validate_row_data_arm80;+#endif+ }+} funcs;++void quantize_row_q4_0_reference(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q4_0_reference(x, y, k);+}++void quantize_row_q4_1_reference(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q4_1_reference(x, y, k);+}++void quantize_row_q5_0_reference(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q5_0_reference(x, y, k);+}++void quantize_row_q5_1_reference(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q5_1_reference(x, y, k);+}++void quantize_row_q8_0_reference(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q8_0_reference(x, y, k);+}++void quantize_row_q8_1_reference(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q8_1_reference(x, y, k);+}++void quantize_row_q2_K_reference(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q2_K_reference(x, y, k);+}++void quantize_row_q3_K_reference(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q3_K_reference(x, y, k);+}++void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q4_K_reference(x, y, k);+}++void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q5_K_reference(x, y, k);+}++void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q6_K_reference(x, y, k);+}++void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q8_K_reference(x, y, k);+}++void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq3_xxs_reference(x, y, k);+}++void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq4_nl_reference(x, y, k);+}++void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq4_xs_reference(x, y, k);+}++void quantize_row_iq3_s_reference (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq3_s_reference(x, y, k);+}++void quantize_row_iq2_s_reference (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq2_s_reference(x, y, k);+}++void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q4_0(x, y, k);+}++void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q4_1(x, y, k);+}++void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q5_0(x, y, k);+}++void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q5_1(x, y, k);+}++void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q8_0(x, y, k);+}++void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q8_1(x, y, k);+}++void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q2_K(x, y, k);+}++void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q3_K(x, y, k);+}++void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q4_K(x, y, k);+}++void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q5_K(x, y, k);+}++void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q6_K(x, y, k);+}++void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_q8_K(x, y, k);+}++void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq3_xxs(x, y, k);+}++void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq4_nl(x, y, k);+}++void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq4_xs(x, y, k);+}++void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq3_s(x, y, k);+}++void quantize_row_iq2_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_quantize_row_iq2_s(x, y, k);+}++void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q4_0(x, y, k);+}++void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q4_1(x, y, k);+}++void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q5_0(x, y, k);+}++void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q5_1(x, y, k);+}++void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q8_0(x, y, k);+}++void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q2_K(x, y, k);+}++void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q3_K(x, y, k);+}++void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q4_K(x, y, k);+}++void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q5_K(x, y, k);+}++void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q6_K(x, y, k);+}++void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_q8_K(x, y, k);+}++void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq2_xxs(x, y, k);+}++void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq2_xs(x, y, k);+}++void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq2_s(x, y, k);+}++void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq3_xxs(x, y, k);+}++void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq1_s(x, y, k);+}++void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq1_m(x, y, k);+}++void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq4_nl(x, y, k);+}++void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq4_xs(x, y, k);+}++void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {+ return funcs.ptr_dequantize_row_iq3_s(x, y, k);+}++void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q4_0_q8_0(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q4_1_q8_1(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q5_0_q8_0(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q5_1_q8_1(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q8_0_q8_0(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q2_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q3_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q4_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q5_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_q6_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq2_xxs_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq2_xs_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq2_s_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq3_xxs_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq1_s_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq1_m_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq4_nl_q8_0(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq4_xs_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_iq3_s_q8_K(n, s, bs, vx, bx, vy, by, nrc);+}++size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq2_xxs(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq2_xs(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq2_s(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq3_xxs(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq1_s(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq1_m(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq4_nl(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq4_xs(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_iq3_s(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q2_K(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q3_K(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q4_K(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q5_K(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q6_K(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q4_0(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q4_1(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q5_0(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q5_1(src, dst, nrows, n_per_row, imatrix);+}++size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {+ return funcs.ptr_quantize_q8_0(src, dst, nrows, n_per_row, imatrix);+}++void iq2xs_init_impl(enum ggml_type type) {+ return funcs.ptr_iq2xs_init_impl(type);+}++void iq2xs_free_impl(enum ggml_type type) {+ return funcs.ptr_iq2xs_free_impl(type);+}++void iq3xs_init_impl(int grid_size) {+ return funcs.ptr_iq3xs_init_impl(grid_size);+}++void iq3xs_free_impl(int grid_size) {+ return funcs.ptr_iq3xs_free_impl(grid_size);+}++bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) {+ return funcs.ptr_ggml_validate_row_data(type, data, nbytes);+}+diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.incsimilarity index 75%rename from llama.cpp/ggml-quants.crename to llama.cpp/ggml-quants.incindex f753167c..5850ff47 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.inc@@ -1,6 +1,3 @@-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi-#define GGML_COMMON_IMPL_C#include "ggml-common.h"@@ -16,98 +13,55 @@#include <float.h>#include <stdlib.h> // for qsort#include <stdio.h> // for GGML_ASSERT-#include <cosmo.h>--#ifdef __ARM_NEON--// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:-//-// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/-//-#include <arm_neon.h>--#else--#ifdef __wasm_simd128__-#include <wasm_simd128.h>-#else-#if defined(__POWER9_VECTOR__) || defined(__powerpc64__)-#include <altivec.h>-#undef bool-#define bool _Bool-#else-#if defined(_MSC_VER) || defined(__MINGW32__)-#include <intrin.h>-#else-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) || defined(__SSE3__)-#if !defined(__riscv)-#include <immintrin.h>-#endif-#endif-#endif-#endif-#endif-#endif--#ifdef __riscv_v_intrinsic-#include <riscv_vector.h>-#endif--#undef MIN-#undef MAX--#define MIN(a, b) ((a) < (b) ? (a) : (b))-#define MAX(a, b) ((a) > (b) ? (a) : (b))#define UNUSED GGML_UNUSED// some compilers don't provide _mm256_set_m128i, e.g. gcc 7#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)+// multiply int8_t, add results pairwise twice+static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {+ // Get absolute values of x vectors+ const __m128i ax = _mm_sign_epi8(x, x);+ // Sign the values of the y vectors+ const __m128i sy = _mm_sign_epi8(y, x);+ // Perform multiplication and create 16-bit values+ const __m128i dot = _mm_maddubs_epi16(ax, sy);+ const __m128i ones = _mm_set1_epi16(1);+ return _mm_madd_epi16(ones, dot);+}++#if __AVX__ || __AVX2__ || __AVX512F__// horizontally add 8 floats-#define hsum_float_8(x) ({ \- __m128 res = _mm256_extractf128_ps(x, 1); \- res = _mm_add_ps(res, _mm256_castps256_ps128(x)); \- res = _mm_add_ps(res, _mm_movehl_ps(res, res)); \- res = _mm_add_ss(res, _mm_movehdup_ps(res)); \- _mm_cvtss_f32(res); \-})--#ifdef __x86_64__--#pragma GCC push_options-#pragma GCC target("avx2")-// Unpack 32 4-bit fields into 32 bytes-// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval-static inline __m256i bytes_from_nibbles_32_avx2(const uint8_t * rsi)-{- const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);- const __m256i lowMask = _mm256_set1_epi8( 0xF );- return _mm256_and_si256(lowMask, bytes);+static inline float hsum_float_8(const __m256 x) {+ __m128 res = _mm256_extractf128_ps(x, 1);+ res = _mm_add_ps(res, _mm256_castps256_ps128(x));+ res = _mm_add_ps(res, _mm_movehl_ps(res, res));+ res = _mm_add_ss(res, _mm_movehdup_ps(res));+ return _mm_cvtss_f32(res);}-#pragma GCC pop_options-#pragma GCC push_options-#pragma GCC target("avx")-// Unpack 32 4-bit fields into 32 bytes-// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval-static inline __m256i bytes_from_nibbles_32_avx(const uint8_t * rsi)-{- // Load 16 bytes from memory- __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);- __m128i tmph = _mm_srli_epi16(tmpl, 4);- const __m128i lowMask = _mm_set1_epi8(0xF);- tmpl = _mm_and_si128(lowMask, tmpl);- tmph = _mm_and_si128(lowMask, tmph);- return MM256_SET_M128I(tmph, tmpl);+// horizontally add 8 int32_t+static inline int hsum_i32_8(const __m256i a) {+ const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));+ const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);+ const __m128i sum64 = _mm_add_epi32(hi64, sum128);+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));+}++// horizontally add 4 int32_t+static inline int hsum_i32_4(const __m128i a) {+ const __m128i hi64 = _mm_unpackhi_epi64(a, a);+ const __m128i sum64 = _mm_add_epi32(hi64, a);+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));}-#pragma GCC pop_options-#pragma GCC push_options-#pragma GCC target("avx2")+#if defined(__AVX2__) || defined(__AVX512F__)// spread 32 bits to 32 bytes { 0x00, 0xFF }-static inline __m256i bytes_from_bits_32_avx2(const uint8_t * x) {+static inline __m256i bytes_from_bits_32(const uint8_t * x) {uint32_t x32;memcpy(&x32, x, sizeof(uint32_t));const __m256i shuf_mask = _mm256_set_epi64x(@@ -118,67 +72,38 @@ static inline __m256i bytes_from_bits_32_avx2(const uint8_t * x) {bytes = _mm256_or_si256(bytes, bit_mask);return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));}-#pragma GCC pop_options--#pragma GCC push_options-#pragma GCC target("avx")-// spread 32 bits to 32 bytes { 0x00, 0xFF }-static inline __m256i bytes_from_bits_32_avx(const uint8_t * x) {- uint32_t x32;- memcpy(&x32, x, sizeof(uint32_t));- const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);- const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);- __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);- __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);- const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);- bytesl = _mm_or_si128(bytesl, bit_mask);- bytesh = _mm_or_si128(bytesh, bit_mask);- bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));- bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));- return MM256_SET_M128I(bytesh, bytesl);-}-#pragma GCC pop_options-// add int16_t pairwise and return as float vector-__attribute__((__target__("avx")))-static inline __m256 sum_i16_pairs_float_avx(const __m128i xh, const __m128i xl) {- const __m128i ones = _mm_set1_epi16(1);- const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);- const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);- const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);- return _mm256_cvtepi32_ps(summed_pairs);+// Unpack 32 4-bit fields into 32 bytes+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)+{+ const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);+ const __m256i lowMask = _mm256_set1_epi8( 0xF );+ return _mm256_and_si256(lowMask, bytes);}// add int16_t pairwise and return as float vector-__attribute__((__target__("avx2,fma")))-static inline __m256 sum_i16_pairs_float_avx2(const __m256i x) {+static inline __m256 sum_i16_pairs_float(const __m256i x) {const __m256i ones = _mm256_set1_epi16(1);const __m256i summed_pairs = _mm256_madd_epi16(ones, x);return _mm256_cvtepi32_ps(summed_pairs);}-__attribute__((__target__("avx2,fma,f16c")))-static inline __m256 mul_sum_us8_pairs_float_avx2(const __m256i ax, const __m256i sy) {+static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {+#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))+ const __m256i zero = _mm256_setzero_si256();+ const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);+ return _mm256_cvtepi32_ps(summed_pairs);+#else// Perform multiplication and create 16-bit valuesconst __m256i dot = _mm256_maddubs_epi16(ax, sy);- return sum_i16_pairs_float_avx2(dot);-}--__attribute__((__target__("avx")))-static inline __m256 mul_sum_us8_pairs_float_avx(const __m256i ax, const __m256i sy) {- const __m128i axl = _mm256_castsi256_si128(ax);- const __m128i axh = _mm256_extractf128_si256(ax, 1);- const __m128i syl = _mm256_castsi256_si128(sy);- const __m128i syh = _mm256_extractf128_si256(sy, 1);- // Perform multiplication and create 16-bit values- const __m128i dotl = _mm_maddubs_epi16(axl, syl);- const __m128i doth = _mm_maddubs_epi16(axh, syh);- return sum_i16_pairs_float_avx(doth, dotl);+ return sum_i16_pairs_float(dot);+#endif}// multiply int8_t, add results pairwise twice and return as float vector-__attribute__((__target__("avx2,fma")))-static inline __m256 mul_sum_i8_pairs_float_avx2(const __m256i x, const __m256i y) {+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {#if __AVXVNNIINT8__const __m256i zero = _mm256_setzero_si256();const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);@@ -188,70 +113,8 @@ static inline __m256 mul_sum_i8_pairs_float_avx2(const __m256i x, const __m256iconst __m256i ax = _mm256_sign_epi8(x, x);// Sign the values of the y vectorsconst __m256i sy = _mm256_sign_epi8(y, x);- return mul_sum_us8_pairs_float_avx2(ax, sy);-#endif-}--// multiply int8_t, add results pairwise twice and return as float vector-__attribute__((__target__("avx")))-static inline __m256 mul_sum_i8_pairs_float_avx(const __m256i x, const __m256i y) {- const __m128i xl = _mm256_castsi256_si128(x);- const __m128i xh = _mm256_extractf128_si256(x, 1);- const __m128i yl = _mm256_castsi256_si128(y);- const __m128i yh = _mm256_extractf128_si256(y, 1);- // Get absolute values of x vectors- const __m128i axl = _mm_sign_epi8(xl, xl);- const __m128i axh = _mm_sign_epi8(xh, xh);- // Sign the values of the y vectors- const __m128i syl = _mm_sign_epi8(yl, xl);- const __m128i syh = _mm_sign_epi8(yh, xh);- // Perform multiplication and create 16-bit values- const __m128i dotl = _mm_maddubs_epi16(axl, syl);- const __m128i doth = _mm_maddubs_epi16(axh, syh);- return sum_i16_pairs_float_avx(doth, dotl);-}-+ return mul_sum_us8_pairs_float(ax, sy);#endif--#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)-// multiply int8_t, add results pairwise twice-static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {- // Get absolute values of x vectors- const __m128i ax = _mm_sign_epi8(x, x);- // Sign the values of the y vectors- const __m128i sy = _mm_sign_epi8(y, x);- // Perform multiplication and create 16-bit values- const __m128i dot = _mm_maddubs_epi16(ax, sy);- const __m128i ones = _mm_set1_epi16(1);- return _mm_madd_epi16(ones, dot);-}--#if __AVX__ || __AVX2__ || __AVX512F__--// horizontally add 8 int32_t-static inline int hsum_i32_8(const __m256i a) {- const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));- const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);- const __m128i sum64 = _mm_add_epi32(hi64, sum128);- const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));- return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));-}--// horizontally add 4 int32_t-static inline int hsum_i32_4(const __m128i a) {- const __m128i hi64 = _mm_unpackhi_epi64(a, a);- const __m128i sum64 = _mm_add_epi32(hi64, a);- const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));- return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));-}--#if defined(__AVX2__) || defined(__AVX512F__)--// add int16_t pairwise and return as float vector-static inline __m256 sum_i16_pairs_float(const __m256i x) {- const __m256i ones = _mm256_set1_epi16(1);- const __m256i summed_pairs = _mm256_madd_epi16(ones, x);- return _mm256_cvtepi32_ps(summed_pairs);}static inline __m128i packNibbles( __m256i bytes )@@ -275,6 +138,34 @@ static inline __m128i packNibbles( __m256i bytes )#endif}#elif defined(__AVX__)+// spread 32 bits to 32 bytes { 0x00, 0xFF }+static inline __m256i bytes_from_bits_32(const uint8_t * x) {+ uint32_t x32;+ memcpy(&x32, x, sizeof(uint32_t));+ const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);+ const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);+ __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);+ __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);+ const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);+ bytesl = _mm_or_si128(bytesl, bit_mask);+ bytesh = _mm_or_si128(bytesh, bit_mask);+ bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));+ bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));+ return MM256_SET_M128I(bytesh, bytesl);+}++// Unpack 32 4-bit fields into 32 bytes+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)+{+ // Load 16 bytes from memory+ __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);+ __m128i tmph = _mm_srli_epi16(tmpl, 4);+ const __m128i lowMask = _mm_set1_epi8(0xF);+ tmpl = _mm_and_si128(lowMask, tmpl);+ tmph = _mm_and_si128(lowMask, tmph);+ return MM256_SET_M128I(tmph, tmpl);+}// add int16_t pairwise and return as float vectorstatic inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {@@ -285,6 +176,35 @@ static inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {return _mm256_cvtepi32_ps(summed_pairs);}+static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {+ const __m128i axl = _mm256_castsi256_si128(ax);+ const __m128i axh = _mm256_extractf128_si256(ax, 1);+ const __m128i syl = _mm256_castsi256_si128(sy);+ const __m128i syh = _mm256_extractf128_si256(sy, 1);+ // Perform multiplication and create 16-bit values+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);+ const __m128i doth = _mm_maddubs_epi16(axh, syh);+ return sum_i16_pairs_float(doth, dotl);+}++// multiply int8_t, add results pairwise twice and return as float vector+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {+ const __m128i xl = _mm256_castsi256_si128(x);+ const __m128i xh = _mm256_extractf128_si256(x, 1);+ const __m128i yl = _mm256_castsi256_si128(y);+ const __m128i yh = _mm256_extractf128_si256(y, 1);+ // Get absolute values of x vectors+ const __m128i axl = _mm_sign_epi8(xl, xl);+ const __m128i axh = _mm_sign_epi8(xh, xh);+ // Sign the values of the y vectors+ const __m128i syl = _mm_sign_epi8(yl, xl);+ const __m128i syh = _mm_sign_epi8(yh, xh);+ // Perform multiplication and create 16-bit values+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);+ const __m128i doth = _mm_maddubs_epi16(axh, syh);+ return sum_i16_pairs_float(doth, dotl);+}+static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 ){// Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh@@ -536,67 +456,214 @@ void quantize_row_q8_0_reference(const float * restrict x, block_q8_0 * restrict}}-#ifdef __x86_64__-#define quantize_row_q8_0 quantize_row_q8_0_default-#include "quantize_row_q8_0.inc"-#undef quantize_row_q8_0-#define quantize_row_q8_0 __attribute__((__target__("avx"))) quantize_row_q8_0_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "quantize_row_q8_0.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef quantize_row_q8_0-#define quantize_row_q8_0 __attribute__((__target__("avx2,fma,f16c"))) quantize_row_q8_0_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "quantize_row_q8_0.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef quantize_row_q8_0void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- quantize_row_q8_0_avx2(x, vy, k);- } else if (X86_HAVE(AVX)) {- quantize_row_q8_0_avx(x, vy, k);- } else {- quantize_row_q8_0_default(x, vy, k);- }-}-#else-#define static-#include "quantize_row_q8_0.inc"-#undef static-#endif+ assert(QK8_0 == 32);+ assert(k % QK8_0 == 0);+ const int nb = k / QK8_0;-// reference implementation for deterministic creation of model files-void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict y, int64_t k) {- assert(QK8_1 == 32);- assert(k % QK8_1 == 0);- const int nb = k / QK8_1;+ block_q8_0 * restrict y = vy;+#if defined(__ARM_NEON)for (int i = 0; i < nb; i++) {- float amax = 0.0f; // absolute max+ float32x4_t srcv [8];+ float32x4_t asrcv[8];+ float32x4_t amaxv[8];- for (int j = 0; j < QK8_1; j++) {- const float v = x[i*QK8_1 + j];- amax = MAX(amax, fabsf(v));- }+ for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);+ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);++ for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);+ for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);+ for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);++ const float amax = vmaxvq_f32(amaxv[0]);const float d = amax / ((1 << 7) - 1);const float id = d ? 1.0f/d : 0.0f;y[i].d = GGML_FP32_TO_FP16(d);- int sum = 0;+ for (int j = 0; j < 8; j++) {+ const float32x4_t v = vmulq_n_f32(srcv[j], id);+ const int32x4_t vi = vcvtnq_s32_f32(v);++ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);+ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);+ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);+ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);+ }+ }+#elif defined(__wasm_simd128__)+ for (int i = 0; i < nb; i++) {+ v128_t srcv [8];+ v128_t asrcv[8];+ v128_t amaxv[8];++ for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);+ for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);++ for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);+ for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);+ for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);++ const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),+ wasm_f32x4_extract_lane(amaxv[0], 1)),+ MAX(wasm_f32x4_extract_lane(amaxv[0], 2),+ wasm_f32x4_extract_lane(amaxv[0], 3)));++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ for (int j = 0; j < 8; j++) {+ const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));+ const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);++ y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);+ y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);+ y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);+ y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);+ }+ }+#elif defined(__AVX2__) || defined(__AVX__)+ for (int i = 0; i < nb; i++) {+ // Load elements into 4 AVX vectors+ __m256 v0 = _mm256_loadu_ps( x );+ __m256 v1 = _mm256_loadu_ps( x + 8 );+ __m256 v2 = _mm256_loadu_ps( x + 16 );+ __m256 v3 = _mm256_loadu_ps( x + 24 );+ x += 32;++ // Compute max(abs(e)) for the block+ const __m256 signBit = _mm256_set1_ps( -0.0f );+ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );++ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );+ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );+ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );+ const float maxScalar = _mm_cvtss_f32( max4 );++ // Quantize these floats+ const float d = maxScalar / 127.f;+ y[i].d = GGML_FP32_TO_FP16(d);+ const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;+ const __m256 mul = _mm256_set1_ps( id );++ // Apply the multiplier+ v0 = _mm256_mul_ps( v0, mul );+ v1 = _mm256_mul_ps( v1, mul );+ v2 = _mm256_mul_ps( v2, mul );+ v3 = _mm256_mul_ps( v3, mul );++ // Round to nearest integer+ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );+ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );+ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );+ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );++ // Convert floats to integers+ __m256i i0 = _mm256_cvtps_epi32( v0 );+ __m256i i1 = _mm256_cvtps_epi32( v1 );+ __m256i i2 = _mm256_cvtps_epi32( v2 );+ __m256i i3 = _mm256_cvtps_epi32( v3 );++#if defined(__AVX2__)+ // Convert int32 to int16+ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15+ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31+ // Convert int16 to int8+ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31++ // We got our precious signed bytes, but the order is now wrong+ // These AVX2 pack instructions process 16-byte pieces independently+ // The following instruction is fixing the order+ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );+ i0 = _mm256_permutevar8x32_epi32( i0, perm );++ _mm256_storeu_si256((__m256i *)y[i].qs, i0);+#else+ // Since we don't have in AVX some necessary functions,+ // we split the registers in half and call AVX2 analogs from SSE+ __m128i ni0 = _mm256_castsi256_si128( i0 );+ __m128i ni1 = _mm256_extractf128_si256( i0, 1);+ __m128i ni2 = _mm256_castsi256_si128( i1 );+ __m128i ni3 = _mm256_extractf128_si256( i1, 1);+ __m128i ni4 = _mm256_castsi256_si128( i2 );+ __m128i ni5 = _mm256_extractf128_si256( i2, 1);+ __m128i ni6 = _mm256_castsi256_si128( i3 );+ __m128i ni7 = _mm256_extractf128_si256( i3, 1);++ // Convert int32 to int16+ ni0 = _mm_packs_epi32( ni0, ni1 );+ ni2 = _mm_packs_epi32( ni2, ni3 );+ ni4 = _mm_packs_epi32( ni4, ni5 );+ ni6 = _mm_packs_epi32( ni6, ni7 );+ // Convert int16 to int8+ ni0 = _mm_packs_epi16( ni0, ni2 );+ ni4 = _mm_packs_epi16( ni4, ni6 );++ _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);+ _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);+#endif+ }+#elif defined(__riscv_v_intrinsic)++ size_t vl = __riscv_vsetvl_e32m4(QK8_0);++ for (int i = 0; i < nb; i++) {+ // load elements+ vfloat32m4_t v_x = __riscv_vle32_v_f32m4(x+i*QK8_0, vl);++ vfloat32m4_t vfabs = __riscv_vfabs_v_f32m4(v_x, vl);+ vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl);+ vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m4_f32m1(vfabs, tmp, vl);+ float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);++ // convert to integer+ vint16m2_t vi = __riscv_vfncvt_x_f_w_i16m2(x0, vl);+ vint8m1_t vs = __riscv_vncvt_x_x_w_i8m1(vi, vl);++ // store result+ __riscv_vse8_v_i8m1(y[i].qs , vs, vl);+ }+#else+ GGML_UNUSED(nb);+ // scalar+ quantize_row_q8_0_reference(x, y, k);+#endif+}++// reference implementation for deterministic creation of model files+void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict y, int64_t k) {+ assert(QK8_1 == 32);+ assert(k % QK8_1 == 0);+ const int nb = k / QK8_1;++ for (int i = 0; i < nb; i++) {+ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK8_1; j++) {+ const float v = x[i*QK8_1 + j];+ amax = MAX(amax, fabsf(v));+ }++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ y[i].d = GGML_FP32_TO_FP16(d);++ int sum = 0;for (int j = 0; j < QK8_1/2; ++j) {const float v0 = x[i*QK8_1 + j]*id;@@ -950,6 +1017,8 @@ void dequantize_row_q8_0(const block_q8_0 * restrict x, float * restrict y, int6// ===================== Helper functions//static inline int nearest_int(float fval) {+ // [jart] https://github.com/ggerganov/llama.cpp/issues/2982+ // assert(fval <= 4194303.f);fval = fminf(fval, 4194303.f);float val = fval + 12582912.f;int i; memcpy(&i, &val, sizeof(int));@@ -1198,8 +1267,8 @@ static float make_qkx2_quants(int n, int nmax, const float * restrict x, const ffor (int i = 0; i < n; ++i) {L[i] = Laux[i];}- best_mad = mad;- scale = this_scale;+ best_mad = mad;+ scale = this_scale;min = this_min;}}@@ -3321,49 +3390,8 @@ static inline __m128i get_scale_shuffle(int i) {}#endif-#ifdef __x86_64__-#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_default-#include "ggml_vec_dot_q4_0_q8_0.inc"-#undef ggml_vec_dot_q4_0_q8_0-#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q4_0_q8_0_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "ggml_vec_dot_q4_0_q8_0.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef ggml_vec_dot_q4_0_q8_0-#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_0_q8_0_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_q4_0_q8_0.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_q4_0_q8_0-void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q4_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);- } else if (X86_HAVE(AVX)) {- ggml_vec_dot_q4_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);- } else {- ggml_vec_dot_q4_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);- }-}-#else-#define static-#include "ggml_vec_dot_q4_0_q8_0.inc"-#undef static-#endif--void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- const int qk = QK8_1;+void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ const int qk = QK8_0;const int nb = n / qk;assert(n % qk == 0);@@ -3377,41 +3405,42 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rUNUSED(by);UNUSED(bs);- const block_q4_1 * restrict x = vx;- const block_q8_1 * restrict y = vy;+ const block_q4_0 * restrict x = vx;+ const block_q8_0 * restrict y = vy;#if defined(__ARM_FEATURE_MATMUL_INT8)if (nrc == 2) {- const block_q4_1 * restrict vx0 = vx;- const block_q4_1 * restrict vx1 = vx + bx;- const block_q8_1 * restrict vy0 = vy;- const block_q8_1 * restrict vy1 = vy + by;+ const block_q4_0 * restrict vx0 = vx;+ const block_q4_0 * restrict vx1 = vx + bx;++ const block_q8_0 * restrict vy0 = vy;+ const block_q8_0 * restrict vy1 = vy + by;float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t summs0 = vdupq_n_f32(0.0f);for (int i = 0; i < nb; i++) {- const block_q4_1 * restrict b_x0 = &vx0[i];- const block_q4_1 * restrict b_x1 = &vx1[i];- const block_q8_1 * restrict b_y0 = &vy0[i];- const block_q8_1 * restrict b_y1 = &vy1[i];-- float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),- GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),- GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),- GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};- summs0 += summs_t;+ const block_q4_0 * restrict b_x0 = &vx0[i];+ const block_q4_0 * restrict b_x1 = &vx1[i];+ const block_q8_0 * restrict b_y0 = &vy0[i];+ const block_q8_0 * restrict b_y1 = &vy1[i];const uint8x16_t m4b = vdupq_n_u8(0x0F);+ const int8x16_t s8b = vdupq_n_s8(0x8);const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);// 4-bit -> 8-bit- const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // sub 8+ const int8x16_t x0_l = vsubq_s8(v0_0l, s8b);+ const int8x16_t x0_h = vsubq_s8(v0_0h, s8b);+ const int8x16_t x1_l = vsubq_s8(v0_1l, s8b);+ const int8x16_t x1_h = vsubq_s8(v0_1h, s8b);// load yconst int8x16_t y0_l = vld1q_s8(b_y0->qs);@@ -3419,11 +3448,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rconst int8x16_t y1_l = vld1q_s8(b_y1->qs);const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);- // mmla into int32x4_t- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,- GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,- GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,- GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};+ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));@@ -3436,37 +3464,32 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rint8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));+sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),l1, r1)), l2, r2)), l3, r3))), scale);}-float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);- sumv2 = sumv2 + summs0;vst1_f32(s, vget_low_f32(sumv2));vst1_f32(s + bs, vget_high_f32(sumv2));return;}#endif- // TODO: add WASM SIMD#if defined(__ARM_NEON)float32x4_t sumv0 = vdupq_n_f32(0.0f);float32x4_t sumv1 = vdupq_n_f32(0.0f);- float summs = 0;-assert(nb % 2 == 0); // TODO: handle odd nbfor (int i = 0; i < nb; i += 2) {- const block_q4_1 * restrict x0 = &x[i + 0];- const block_q4_1 * restrict x1 = &x[i + 1];- const block_q8_1 * restrict y0 = &y[i + 0];- const block_q8_1 * restrict y1 = &y[i + 1];-- summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);+ const block_q4_0 * restrict x0 = &x[i + 0];+ const block_q4_0 * restrict x1 = &x[i + 1];+ const block_q8_0 * restrict y0 = &y[i + 0];+ const block_q8_0 * restrict y1 = &y[i + 1];const uint8x16_t m4b = vdupq_n_u8(0x0F);+ const int8x16_t s8b = vdupq_n_s8(0x8);const uint8x16_t v0_0 = vld1q_u8(x0->qs);const uint8x16_t v0_1 = vld1q_u8(x1->qs);@@ -3477,6 +3500,12 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rconst int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));+ // sub 8+ const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);+ const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);+ const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);+ const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);+// load yconst int8x16_t v1_0l = vld1q_s8(y0->qs);const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);@@ -3484,56 +3513,192 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rconst int8x16_t v1_1h = vld1q_s8(y1->qs + 16);// dot product into int32x4_t- const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);- const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));}- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;-#elif defined(__AVX2__) || defined(__AVX__)+ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);+#elif defined(__AVX2__)// Initialize accumulator with zeros__m256 acc = _mm256_setzero_ps();- float summs = 0;-// Main loopfor (int i = 0; i < nb; ++i) {- const float d0 = GGML_FP16_TO_FP32(x[i].d);- const float d1 = GGML_FP16_TO_FP32(y[i].d);+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );- summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);+ __m256i qx = bytes_from_nibbles_32(x[i].qs);- const __m256 d0v = _mm256_set1_ps( d0 );- const __m256 d1v = _mm256_set1_ps( d1 );+ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.+ const __m256i off = _mm256_set1_epi8( 8 );+ qx = _mm256_sub_epi8( qx, off );- // Compute combined scales- const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);- // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes-#ifdef __AVX2__- const __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);-#else- const __m256i qx = bytes_from_nibbles_32_avx(x[i].qs);-#endif- const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[i].qs );+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);-#if defined(__AVX2__)- const __m256 xy = mul_sum_us8_pairs_float_avx2(qx, qy);-#else- const __m256 xy = mul_sum_us8_pairs_float_avx(qx, qy);-#endif+ /* Multiply q with scale and accumulate */+ acc = _mm256_fmadd_ps( d, q, acc );+ }- // Accumulate d0*d1*x*y-#if defined(__AVX2__)- acc = _mm256_fmadd_ps( d0d1, xy, acc );-#else- acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );-#endif+ *s = hsum_float_8(acc);+#elif defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (int i = 0; i < nb; ++i) {+ // Compute combined scale for the block+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );++ const __m128i lowMask = _mm_set1_epi8(0xF);+ const __m128i off = _mm_set1_epi8(8);++ const __m128i tmp = _mm_loadu_si128((const __m128i *)x[i].qs);++ __m128i bx_0 = _mm_and_si128(lowMask, tmp);+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);+ bx_0 = _mm_sub_epi8(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ bx_0 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4));+ by_0 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));+ bx_0 = _mm_sub_epi8(bx_0, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_0, by_0);++ // Convert int32_t to float+ __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1));++ // Apply the scale, and accumulate+ acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);}- *s = hsum_float_8(acc) + summs;+ *s = hsum_float_8(acc);+#elif defined(__SSSE3__)+ // set constants+ const __m128i lowMask = _mm_set1_epi8(0xF);+ const __m128i off = _mm_set1_epi8(8);++ // Initialize accumulator with zeros+ __m128 acc_0 = _mm_setzero_ps();+ __m128 acc_1 = _mm_setzero_ps();+ __m128 acc_2 = _mm_setzero_ps();+ __m128 acc_3 = _mm_setzero_ps();++ // First round without accumulation+ {+ _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 0 and 1+ const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) );++ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[0].qs);++ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[0].qs);+ bx_0 = _mm_sub_epi8(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[0].qs + 16));+ bx_1 = _mm_sub_epi8(bx_1, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);++ _mm_prefetch(&x[1] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[1] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 2 and 3+ const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) );++ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[1].qs);++ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[1].qs);+ bx_2 = _mm_sub_epi8(bx_2, off);+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);++ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[1].qs + 16));+ bx_3 = _mm_sub_epi8(bx_3, off);+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);++ // Convert int32_t to float+ __m128 p0 = _mm_cvtepi32_ps(i32_0);+ __m128 p1 = _mm_cvtepi32_ps(i32_1);+ __m128 p2 = _mm_cvtepi32_ps(i32_2);+ __m128 p3 = _mm_cvtepi32_ps(i32_3);++ // Apply the scale+ acc_0 = _mm_mul_ps( d_0_1, p0 );+ acc_1 = _mm_mul_ps( d_0_1, p1 );+ acc_2 = _mm_mul_ps( d_2_3, p2 );+ acc_3 = _mm_mul_ps( d_2_3, p3 );+ }++ assert(nb % 2 == 0); // TODO: handle odd nb++ // Main loop+ for (int i = 2; i < nb; i+=2) {+ _mm_prefetch(&x[i] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[i] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 0 and 1+ const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );++ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[i].qs);++ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);+ bx_0 = _mm_sub_epi8(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));+ bx_1 = _mm_sub_epi8(bx_1, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);++ _mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 2 and 3+ const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) );++ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[i + 1].qs);++ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[i + 1].qs);+ bx_2 = _mm_sub_epi8(bx_2, off);+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);++ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[i + 1].qs + 16));+ bx_3 = _mm_sub_epi8(bx_3, off);+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);++ // Convert int32_t to float+ __m128 p0 = _mm_cvtepi32_ps(i32_0);+ __m128 p1 = _mm_cvtepi32_ps(i32_1);+ __m128 p2 = _mm_cvtepi32_ps(i32_2);+ __m128 p3 = _mm_cvtepi32_ps(i32_3);++ // Apply the scale+ __m128 p0_d = _mm_mul_ps( d_0_1, p0 );+ __m128 p1_d = _mm_mul_ps( d_0_1, p1 );+ __m128 p2_d = _mm_mul_ps( d_2_3, p2 );+ __m128 p3_d = _mm_mul_ps( d_2_3, p3 );++ // Acummulate+ acc_0 = _mm_add_ps(p0_d, acc_0);+ acc_1 = _mm_add_ps(p1_d, acc_1);+ acc_2 = _mm_add_ps(p2_d, acc_2);+ acc_3 = _mm_add_ps(p3_d, acc_3);+ }++ *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);#elif defined(__riscv_v_intrinsic)float sumf = 0.0;@@ -3550,8 +3715,12 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rvuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);- vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);+ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ // subtract offset+ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 8, vl);+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 8, vl);vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);@@ -3561,9 +3730,9 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rvint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2)mul_sum_us8_pairs_float;- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);}*s = sumf;@@ -3575,119 +3744,124 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rint sumi = 0;for (int j = 0; j < qk/2; ++j) {- const int v0 = (x[i].qs[j] & 0x0F);- const int v1 = (x[i].qs[j] >> 4);+ const int v0 = (x[i].qs[j] & 0x0F) - 8;+ const int v1 = (x[i].qs[j] >> 4) - 8;sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);}- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);}*s = sumf;#endif}-#ifdef __x86_64__-#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_default-#include "ggml_vec_dot_q5_0_q8_0.inc"-#undef ggml_vec_dot_q5_0_q8_0-#define ggml_vec_dot_q5_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q5_0_q8_0_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "ggml_vec_dot_q5_0_q8_0.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef ggml_vec_dot_q5_0_q8_0-#define ggml_vec_dot_q5_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q5_0_q8_0_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_q5_0_q8_0.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_q5_0_q8_0-void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q5_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);- } else if (X86_HAVE(AVX)) {- ggml_vec_dot_q5_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);- } else {- ggml_vec_dot_q5_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);- }-}-#else-#define static-#include "ggml_vec_dot_q5_0_q8_0.inc"-#undef static-#endif--void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const int qk = QK8_1;const int nb = n / qk;assert(n % qk == 0);- assert(qk == QK5_1);+#if defined(__ARM_FEATURE_MATMUL_INT8)+ assert((nrc == 2) || (nrc == 1));+#elseassert(nrc == 1);+#endifUNUSED(nrc);UNUSED(bx);UNUSED(by);UNUSED(bs);- const block_q5_1 * restrict x = vx;+ const block_q4_1 * restrict x = vx;const block_q8_1 * restrict y = vy;-#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);+#if defined(__ARM_FEATURE_MATMUL_INT8)+ if (nrc == 2) {+ const block_q4_1 * restrict vx0 = vx;+ const block_q4_1 * restrict vx1 = vx + bx;+ const block_q8_1 * restrict vy0 = vy;+ const block_q8_1 * restrict vy1 = vy + by;- float summs0 = 0.0f;- float summs1 = 0.0f;+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t summs0 = vdupq_n_f32(0.0f);- uint32_t qh0;- uint32_t qh1;+ for (int i = 0; i < nb; i++) {+ const block_q4_1 * restrict b_x0 = &vx0[i];+ const block_q4_1 * restrict b_x1 = &vx1[i];+ const block_q8_1 * restrict b_y0 = &vy0[i];+ const block_q8_1 * restrict b_y1 = &vy1[i];- uint64_t tmp0[4];- uint64_t tmp1[4];+ float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),+ GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};+ summs0 += summs_t;- assert(nb % 2 == 0); // TODO: handle odd nb+ const uint8x16_t m4b = vdupq_n_u8(0x0F);- for (int i = 0; i < nb; i += 2) {- const block_q5_1 * restrict x0 = &x[i];- const block_q5_1 * restrict x1 = &x[i + 1];- const block_q8_1 * restrict y0 = &y[i];- const block_q8_1 * restrict y1 = &y[i + 1];+ const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);- const uint8x16_t m4b = vdupq_n_u8(0x0F);+ // 4-bit -> 8-bit+ const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));- summs0 += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);- summs1 += GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);+ // load y+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);- // extract the 5th bit via lookup table ((b) << 4)- memcpy(&qh0, x0->qh, sizeof(qh0));- memcpy(&qh1, x1->qh, sizeof(qh1));+ // mmla into int32x4_t+ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,+ GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,+ GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,+ GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};- tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];- tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];- tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];- tmp0[3] = table_b2b_0[(qh0 >> 24) ];+ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));- tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];- tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];- tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];- tmp1[3] = table_b2b_0[(qh1 >> 24) ];+ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));- const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));- const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));- const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));- const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));+ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));++ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),+ l1, r1)), l2, r2)), l3, r3))), scale);+ }++ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);+ sumv2 = sumv2 + summs0;++ vst1_f32(s, vget_low_f32(sumv2));+ vst1_f32(s + bs, vget_high_f32(sumv2));+ return;+ }+#endif+ // TODO: add WASM SIMD+#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ float summs = 0;++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q4_1 * restrict x0 = &x[i + 0];+ const block_q4_1 * restrict x1 = &x[i + 1];+ const block_q8_1 * restrict y0 = &y[i + 0];+ const block_q8_1 * restrict y1 = &y[i + 1];++ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);++ const uint8x16_t m4b = vdupq_n_u8(0x0F);const uint8x16_t v0_0 = vld1q_u8(x0->qs);const uint8x16_t v0_1 = vld1q_u8(x1->qs);@@ -3698,64 +3872,226 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * rconst int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));- // add high bit- const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);- const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);- const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);- const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);-// load yconst int8x16_t v1_0l = vld1q_s8(y0->qs);const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);const int8x16_t v1_1l = vld1q_s8(y1->qs);const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),- ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),- ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));- }+ // dot product into int32x4_t+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;-#elif defined(__wasm_simd128__)- v128_t sumv = wasm_f32x4_splat(0.0f);+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+ }- float summs = 0.0f;+ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;+#elif defined(__AVX2__) || defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();- uint32_t qh;- uint64_t tmp[4];+ float summs = 0;- // TODO: check if unrolling this is better+ // Main loopfor (int i = 0; i < nb; ++i) {- const block_q5_1 * restrict x0 = &x[i];- const block_q8_1 * restrict y0 = &y[i];-- summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);+ const float d0 = GGML_FP16_TO_FP32(x[i].d);+ const float d1 = GGML_FP16_TO_FP32(y[i].d);- const v128_t m4b = wasm_i8x16_splat(0x0F);+ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);- // extract the 5th bit- memcpy(&qh, x0->qh, sizeof(qh));+ const __m256 d0v = _mm256_set1_ps( d0 );+ const __m256 d1v = _mm256_set1_ps( d1 );- tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];- tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];- tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];- tmp[3] = table_b2b_0[(qh >> 24) ];+ // Compute combined scales+ const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );- const v128_t qhl = wasm_v128_load(tmp + 0);- const v128_t qhh = wasm_v128_load(tmp + 2);+ // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes+ const __m256i qx = bytes_from_nibbles_32(x[i].qs);+ const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[i].qs );- const v128_t v0 = wasm_v128_load(x0->qs);+ const __m256 xy = mul_sum_us8_pairs_float(qx, qy);- // 4-bit -> 8-bit- const v128_t v0l = wasm_v128_and (v0, m4b);- const v128_t v0h = wasm_u8x16_shr(v0, 4);+ // Accumulate d0*d1*x*y+#if defined(__AVX2__)+ acc = _mm256_fmadd_ps( d0d1, xy, acc );+#else+ acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );+#endif+ }- // add high bit- const v128_t v0lf = wasm_v128_or(v0l, qhl);- const v128_t v0hf = wasm_v128_or(v0h, qhh);+ *s = hsum_float_8(acc) + summs;+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;++ size_t vl = __riscv_vsetvl_e8m1(qk/2);++ for (int i = 0; i < nb; i++) {+ // load elements+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);++ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);++ // mask and store lower part of x, and then upper part+ vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);+ vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);++ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);++ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);++ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);++ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ int sumi = 0;++ for (int j = 0; j < qk/2; ++j) {+ const int v0 = (x[i].qs[j] & 0x0F);+ const int v1 = (x[i].qs[j] >> 4);++ sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);+ }++ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ }++ *s = sumf;+#endif+}++void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(qk == QK5_0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q5_0 * restrict x = vx;+ const block_q8_0 * restrict y = vy;++#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ uint32_t qh0;+ uint32_t qh1;++ uint64_t tmp0[4];+ uint64_t tmp1[4];++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q5_0 * restrict x0 = &x[i];+ const block_q5_0 * restrict x1 = &x[i + 1];+ const block_q8_0 * restrict y0 = &y[i];+ const block_q8_0 * restrict y1 = &y[i + 1];++ const uint8x16_t m4b = vdupq_n_u8(0x0F);++ // extract the 5th bit via lookup table ((!b) << 4)+ memcpy(&qh0, x0->qh, sizeof(qh0));+ memcpy(&qh1, x1->qh, sizeof(qh1));++ tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];+ tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];+ tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];+ tmp0[3] = table_b2b_1[(qh0 >> 24) ];++ tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];+ tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];+ tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];+ tmp1[3] = table_b2b_1[(qh1 >> 24) ];++ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));++ const uint8x16_t v0_0 = vld1q_u8(x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);++ // 4-bit -> 8-bit+ int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)+ const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);+ const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);+ const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);+ const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);++ // load y+ const int8x16_t v1_0l = vld1q_s8(y0->qs);+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);+ const int8x16_t v1_1l = vld1q_s8(y1->qs);+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);+#elif defined(__wasm_simd128__)+ v128_t sumv = wasm_f32x4_splat(0.0f);++ uint32_t qh;+ uint64_t tmp[4];++ // TODO: check if unrolling this is better+ for (int i = 0; i < nb; ++i) {+ const block_q5_0 * restrict x0 = &x[i];+ const block_q8_0 * restrict y0 = &y[i];++ const v128_t m4b = wasm_i8x16_splat(0x0F);++ // extract the 5th bit+ memcpy(&qh, x0->qh, sizeof(qh));++ tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];+ tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];+ tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];+ tmp[3] = table_b2b_1[(qh >> 24) ];++ const v128_t qhl = wasm_v128_load(tmp + 0);+ const v128_t qhh = wasm_v128_load(tmp + 2);++ const v128_t v0 = wasm_v128_load(x0->qs);++ // 4-bit -> 8-bit+ const v128_t v0l = wasm_v128_and (v0, m4b);+ const v128_t v0h = wasm_u8x16_shr(v0, 4);++ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)+ const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);+ const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);// load yconst v128_t v1l = wasm_v128_load(y0->qs);@@ -3773,8 +4109,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * rconst v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);// dot product- sumv = wasm_f32x4_add(sumv,- wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(+ sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(+ wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),wasm_i32x4_dot_i16x8(v0lfh, v1lh)),wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),@@ -3783,67 +4119,61 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r}*s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +- wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;+ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);#elif defined(__AVX2__)// Initialize accumulator with zeros__m256 acc = _mm256_setzero_ps();- float summs = 0.0f;-// Main loopfor (int i = 0; i < nb; i++) {- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));-- summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));- __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);- __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);- bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));+ __m256i qx = bytes_from_nibbles_32(x[i].qs);+ __m256i bxhi = bytes_from_bits_32(x[i].qh);+ bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));qx = _mm256_or_si256(qx, bxhi);- const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));- const __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);- const __m256 q = mul_sum_us8_pairs_float_avx2(qx, qy);+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);- acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);+ /* Multiply q with scale and accumulate */+ acc = _mm256_fmadd_ps(d, q, acc);}- *s = hsum_float_8(acc) + summs;+ *s = hsum_float_8(acc);#elif defined(__AVX__)// Initialize accumulator with zeros__m256 acc = _mm256_setzero_ps();- __m128i mask = _mm_set1_epi8(0x10);-- float summs = 0.0f;+ __m128i mask = _mm_set1_epi8((char)0xF0);// Main loopfor (int i = 0; i < nb; i++) {- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));-- summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));- __m256i bx_0 = bytes_from_nibbles_32_avx(x[i].qs);- const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);+ __m256i bx_0 = bytes_from_nibbles_32(x[i].qs);+ const __m256i bxhi = bytes_from_bits_32(x[i].qh);__m128i bxhil = _mm256_castsi256_si128(bxhi);__m128i bxhih = _mm256_extractf128_si256(bxhi, 1);- bxhil = _mm_and_si128(bxhil, mask);- bxhih = _mm_and_si128(bxhih, mask);+ bxhil = _mm_andnot_si128(bxhil, mask);+ bxhih = _mm_andnot_si128(bxhih, mask);__m128i bxl = _mm256_castsi256_si128(bx_0);__m128i bxh = _mm256_extractf128_si256(bx_0, 1);bxl = _mm_or_si128(bxl, bxhil);bxh = _mm_or_si128(bxh, bxhih);bx_0 = MM256_SET_M128I(bxh, bxl);- const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs);- const __m256 q = mul_sum_us8_pairs_float_avx(bx_0, by_0);+ const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0);- acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);+ /* Multiply q with scale and accumulate */+ acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);}- *s = hsum_float_8(acc) + summs;+ *s = hsum_float_8(acc);#elif defined(__riscv_v_intrinsic)float sumf = 0.0;@@ -3851,30 +4181,30 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * rsize_t vl = __riscv_vsetvl_e8m1(qk/2);- // temporary registers for shift operations+ // These temporary registers are for masking and shift operationsvuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);- vuint32m2_t vt_2 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);+ vuint32m2_t vt_2 = __riscv_vsll_vv_u32m2(__riscv_vmv_v_x_u32m2(1, vl), vt_1, vl);++ vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);+ vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);for (int i = 0; i < nb; i++) {memcpy(&qh, x[i].qh, sizeof(uint32_t));- // load qh- vuint32m2_t vqh = __riscv_vmv_v_x_u32m2(qh, vl);-- // ((qh >> (j + 0)) << 4) & 0x10;- vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(vqh, vt_1, vl);+ // ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(xha_0, vt_1, vl);vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);- vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(xhl_0, 0x10, vl);- // ((qh >> (j + 12)) ) & 0x10;- vuint32m2_t xhr_1 = __riscv_vsrl_vv_u32m2(vqh, vt_2, vl);- vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(xhr_1, 0x10, vl);+ // ((qh & (1u << (j + 16))) >> (j + 12));+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(vt_3, qh, vl);+ vuint32m2_t xhl_1 = __riscv_vsrl_vv_u32m2(xha_1, vt_4, vl);// narrowing- vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xha_0, vl);+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xhl_0, vl);vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);- vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xha_1, vl);+ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xhl_1, vl);vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);// load@@ -3889,8 +4219,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * rvuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);- vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);+ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 16, vl);+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 16, vl);vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);@@ -3902,7 +4235,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * rint sumi = __riscv_vmv_x_s_i32m1_i32(vs2);- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;}*s = sumf;@@ -3917,395 +4250,2306 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * rint sumi = 0;for (int j = 0; j < qk/2; ++j) {- const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;- const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;+ const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;+ const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));- const int32_t x0 = (x[i].qs[j] & 0xF) | xh_0;- const int32_t x1 = (x[i].qs[j] >> 4) | xh_1;+ const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;+ const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);}- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;}*s = sumf;#endif}-#ifdef __x86_64__-#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_default-#include "ggml_vec_dot_q8_0_q8_0.inc"-#undef ggml_vec_dot_q8_0_q8_0-#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q8_0_q8_0_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "ggml_vec_dot_q8_0_q8_0.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef ggml_vec_dot_q8_0_q8_0-#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q8_0_q8_0_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_q8_0_q8_0.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_q8_0_q8_0-void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {- ggml_vec_dot_q8_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);- } else if (X86_HAVE(AVX)) {- ggml_vec_dot_q8_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);- } else {- ggml_vec_dot_q8_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);- }-}-#else-#define static-#include "ggml_vec_dot_q8_0_q8_0.inc"-#undef static-#endif+void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ const int qk = QK8_1;+ const int nb = n / qk;-#if QK_K == 256-void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % qk == 0);+ assert(qk == QK5_1);assert(nrc == 1);UNUSED(nrc);UNUSED(bx);UNUSED(by);UNUSED(bs);- const block_q2_K * restrict x = vx;- const block_q8_K * restrict y = vy;+ const block_q5_1 * restrict x = vx;+ const block_q8_1 * restrict y = vy;- const int nb = n / QK_K;+#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);-#ifdef __ARM_NEON- const uint8x16_t m3 = vdupq_n_u8(0x3);- const uint8x16_t m4 = vdupq_n_u8(0xF);+ float summs0 = 0.0f;+ float summs1 = 0.0f;- const int32x4_t vzero = vdupq_n_s32(0);+ uint32_t qh0;+ uint32_t qh1;- ggml_int8x16x2_t q2bytes;- uint8_t aux[16];+ uint64_t tmp0[4];+ uint64_t tmp1[4];- float sum = 0;+ assert(nb % 2 == 0); // TODO: handle odd nb- for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ for (int i = 0; i < nb; i += 2) {+ const block_q5_1 * restrict x0 = &x[i];+ const block_q5_1 * restrict x1 = &x[i + 1];+ const block_q8_1 * restrict y0 = &y[i];+ const block_q8_1 * restrict y1 = &y[i + 1];- const uint8_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- const uint8_t * restrict sc = x[i].scales;+ const uint8x16_t m4b = vdupq_n_u8(0x0F);- const uint8x16_t mins_and_scales = vld1q_u8(sc);- const uint8x16_t scales = vandq_u8(mins_and_scales, m4);- vst1q_u8(aux, scales);+ summs0 += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);+ summs1 += GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);- const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);- const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);- const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};- const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),- vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));- const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),- vmull_s16(vget_high_s16(mins16.val[1]), vget_high_s16(q8sums.val[1])));- sum += dmin * vaddvq_s32(vaddq_s32(s0, s1));+ // extract the 5th bit via lookup table ((b) << 4)+ memcpy(&qh0, x0->qh, sizeof(qh0));+ memcpy(&qh1, x1->qh, sizeof(qh1));- int isum = 0;- int is = 0;+ tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];+ tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];+ tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];+ tmp0[3] = table_b2b_0[(qh0 >> 24) ];++ tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];+ tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];+ tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];+ tmp1[3] = table_b2b_0[(qh1 >> 24) ];++ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));++ const uint8x16_t v0_0 = vld1q_u8(x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);++ // 4-bit -> 8-bit+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // add high bit+ const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);+ const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);+ const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);+ const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);++ // load y+ const int8x16_t v1_0l = vld1q_s8(y0->qs);+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);+ const int8x16_t v1_1l = vld1q_s8(y1->qs);+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;+#elif defined(__wasm_simd128__)+ v128_t sumv = wasm_f32x4_splat(0.0f);++ float summs = 0.0f;++ uint32_t qh;+ uint64_t tmp[4];++ // TODO: check if unrolling this is better+ for (int i = 0; i < nb; ++i) {+ const block_q5_1 * restrict x0 = &x[i];+ const block_q8_1 * restrict y0 = &y[i];++ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);++ const v128_t m4b = wasm_i8x16_splat(0x0F);++ // extract the 5th bit+ memcpy(&qh, x0->qh, sizeof(qh));++ tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];+ tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];+ tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];+ tmp[3] = table_b2b_0[(qh >> 24) ];++ const v128_t qhl = wasm_v128_load(tmp + 0);+ const v128_t qhh = wasm_v128_load(tmp + 2);++ const v128_t v0 = wasm_v128_load(x0->qs);++ // 4-bit -> 8-bit+ const v128_t v0l = wasm_v128_and (v0, m4b);+ const v128_t v0h = wasm_u8x16_shr(v0, 4);++ // add high bit+ const v128_t v0lf = wasm_v128_or(v0l, qhl);+ const v128_t v0hf = wasm_v128_or(v0h, qhh);++ // load y+ const v128_t v1l = wasm_v128_load(y0->qs);+ const v128_t v1h = wasm_v128_load(y0->qs + 16);++ // int8x16 -> int16x8+ const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);+ const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);+ const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);+ const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);++ const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);+ const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);+ const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);+ const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);++ // dot product+ sumv = wasm_f32x4_add(sumv,+ wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),+ wasm_i32x4_dot_i16x8(v0lfh, v1lh)),+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),+ wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),+ wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));+ }++ *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) ++ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;+#elif defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ float summs = 0.0f;++ // Main loop+ for (int i = 0; i < nb; i++) {+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));++ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);++ __m256i qx = bytes_from_nibbles_32(x[i].qs);+ __m256i bxhi = bytes_from_bits_32(x[i].qh);+ bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));+ qx = _mm256_or_si256(qx, bxhi);++ const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));+ const __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_us8_pairs_float(qx, qy);++ acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);+ }++ *s = hsum_float_8(acc) + summs;+#elif defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();+ __m128i mask = _mm_set1_epi8(0x10);++ float summs = 0.0f;++ // Main loop+ for (int i = 0; i < nb; i++) {+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));++ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);++ __m256i bx_0 = bytes_from_nibbles_32(x[i].qs);+ const __m256i bxhi = bytes_from_bits_32(x[i].qh);+ __m128i bxhil = _mm256_castsi256_si128(bxhi);+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);+ bxhil = _mm_and_si128(bxhil, mask);+ bxhih = _mm_and_si128(bxhih, mask);+ __m128i bxl = _mm256_castsi256_si128(bx_0);+ __m128i bxh = _mm256_extractf128_si256(bx_0, 1);+ bxl = _mm_or_si128(bxl, bxhil);+ bxh = _mm_or_si128(bxh, bxhih);+ bx_0 = MM256_SET_M128I(bxh, bxl);++ const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));+ const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0);++ acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);+ }++ *s = hsum_float_8(acc) + summs;+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;++ uint32_t qh;++ size_t vl = __riscv_vsetvl_e8m1(qk/2);++ // temporary registers for shift operations+ vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);+ vuint32m2_t vt_2 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);++ for (int i = 0; i < nb; i++) {+ memcpy(&qh, x[i].qh, sizeof(uint32_t));++ // load qh+ vuint32m2_t vqh = __riscv_vmv_v_x_u32m2(qh, vl);++ // ((qh >> (j + 0)) << 4) & 0x10;+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(vqh, vt_1, vl);+ vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(xhl_0, 0x10, vl);++ // ((qh >> (j + 12)) ) & 0x10;+ vuint32m2_t xhr_1 = __riscv_vsrl_vv_u32m2(vqh, vt_2, vl);+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(xhr_1, 0x10, vl);++ // narrowing+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xha_0, vl);+ vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);++ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xha_1, vl);+ vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);++ // load+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);++ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);++ vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);+ vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);++ vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);+ vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);++ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);++ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);++ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);++ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ uint32_t qh;+ memcpy(&qh, x[i].qh, sizeof(qh));++ int sumi = 0;++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;++ const int32_t x0 = (x[i].qs[j] & 0xF) | xh_0;+ const int32_t x1 = (x[i].qs[j] >> 4) | xh_1;++ sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);+ }++ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);+ }++ *s = sumf;+#endif+}++void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);+#if defined(__ARM_FEATURE_MATMUL_INT8)+ assert((nrc == 2) || (nrc == 1));+#else+ assert(nrc == 1);+#endif+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q8_0 * restrict x = vx;+ const block_q8_0 * restrict y = vy;++#if defined(__ARM_FEATURE_MATMUL_INT8)+ if (nrc == 2) {+ const block_q8_0 * restrict vx0 = vx;+ const block_q8_0 * restrict vx1 = vx + bx;+ const block_q8_0 * restrict vy0 = vy;+ const block_q8_0 * restrict vy1 = vy + by;++ float32x4_t sumv0 = vdupq_n_f32(0.0f);++ for (int i = 0; i < nb; i++) {+ const block_q8_0 * restrict b_x0 = &vx0[i];+ const block_q8_0 * restrict b_y0 = &vy0[i];++ const block_q8_0 * restrict b_x1 = &vx1[i];+ const block_q8_0 * restrict b_y1 = &vy1[i];++ const int8x16_t x0_l = vld1q_s8(b_x0->qs);+ const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16);+ const int8x16_t x1_l = vld1q_s8(b_x1->qs);+ const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16);++ // load y+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);++ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};++ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));++ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));++ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));++ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));++ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),+ l1, r1)), l2, r2)), l3, r3))), scale);+ }+ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);++ vst1_f32(s, vget_low_f32(sumv2));+ vst1_f32(s + bs, vget_high_f32(sumv2));+ return;+ }+#endif+#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q8_0 * restrict x0 = &x[i + 0];+ const block_q8_0 * restrict x1 = &x[i + 1];+ const block_q8_0 * restrict y0 = &y[i + 0];+ const block_q8_0 * restrict y1 = &y[i + 1];++ const int8x16_t x0_0 = vld1q_s8(x0->qs);+ const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);+ const int8x16_t x1_0 = vld1q_s8(x1->qs);+ const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);++ // load y+ const int8x16_t y0_0 = vld1q_s8(y0->qs);+ const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);+ const int8x16_t y1_0 = vld1q_s8(y1->qs);+ const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(+ ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),+ ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));++ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(+ ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),+ ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);+#elif defined(__AVX2__) || defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (int i = 0; i < nb; ++i) {+ // Compute combined scale for the block+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));+ __m256i qx = _mm256_loadu_si256((const __m256i *)x[i].qs);+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_i8_pairs_float(qx, qy);++ // Multiply q with scale and accumulate+#if defined(__AVX2__)+ acc = _mm256_fmadd_ps( d, q, acc );+#else+ acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );+#endif+ }++ *s = hsum_float_8(acc);+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;+ size_t vl = __riscv_vsetvl_e8m1(qk);++ for (int i = 0; i < nb; i++) {+ // load elements+ vint8m1_t bx_0 = __riscv_vle8_v_i8m1(x[i].qs, vl);+ vint8m1_t by_0 = __riscv_vle8_v_i8m1(y[i].qs, vl);++ vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx_0, by_0, vl);++ vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);+ vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);++ sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ int sumi = 0;++ for (int j = 0; j < qk; j++) {+ sumi += x[i].qs[j]*y[i].qs[j];+ }++ sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));+ }++ *s = sumf;+#endif+}++#if QK_K == 256+void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_K * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#ifdef __ARM_NEON+ const uint8x16_t m3 = vdupq_n_u8(0x3);+ const uint8x16_t m4 = vdupq_n_u8(0xF);++ const int32x4_t vzero = vdupq_n_s32(0);++ ggml_int8x16x2_t q2bytes;+ uint8_t aux[16];++ float sum = 0;++ for (int i = 0; i < nb; ++i) {+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ const uint8_t * restrict sc = x[i].scales;++ const uint8x16_t mins_and_scales = vld1q_u8(sc);+ const uint8x16_t scales = vandq_u8(mins_and_scales, m4);+ vst1q_u8(aux, scales);++ const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);+ const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};+ const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),+ vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));+ const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),+ vmull_s16(vget_high_s16(mins16.val[1]), vget_high_s16(q8sums.val[1])));+ sum += dmin * vaddvq_s32(vaddq_s32(s0, s1));++ int isum = 0;+ int is = 0;++// We use this macro instead of a function call because for some reason+// the code runs 2-3% slower, even if the function is declared inline+#define MULTIPLY_ACCUM_WITH_SCALE(index)\+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];++#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\+ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\+ MULTIPLY_ACCUM_WITH_SCALE((index));++ for (int j = 0; j < QK_K/128; ++j) {+ const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;++ ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;+ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));++ MULTIPLY_ACCUM_WITH_SCALE(0);++ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);++ is += 8;+ }++ sum += d * isum;+ }++ *s = sum;++#elif defined __AVX2__++ const __m256i m3 = _mm256_set1_epi8(3);+ const __m128i m4 = _mm_set1_epi8(0xF);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);+ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);+ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);+ const __m256i mins = _mm256_cvtepi8_epi16(mins8);+ const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));++ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);++ const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};++ __m256i sumi = _mm256_setzero_si256();++ for (int j = 0; j < QK_K/128; ++j) {++ const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ const __m256i q2_0 = _mm256_and_si256(q2bits, m3);+ const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);+ const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);+ const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);++ __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);+ __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);+ __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);+ __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);++ p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);+ p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);+ p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);+ p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);++ p0 = _mm256_add_epi32(p0, p1);+ p2 = _mm256_add_epi32(p2, p3);++ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));+ }++ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);++ }++ *s = hsum_float_8(acc);++#elif defined __AVX__++ const __m128i m3 = _mm_set1_epi8(0x3);+ const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i m2 = _mm_set1_epi8(0x2);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ // load mins and scales from block_q2_K.scales[QK_K/16]+ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);+ const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);+ const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);+ const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);+ const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));++ // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2+ const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));+ const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));++ // sumf += -dmin * summs in 32bits*8+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);++ const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));+ const __m128i scales[2] = { scales_0, scales_1 };++ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ for (int j = 0; j < QK_K/128; ++j) {++ // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;++ // load 2bits*16*8 from block_q2_K.qs[QK_K/4]+ __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;+ const __m128i q2_0 = _mm_and_si128(q2bits, m3);+ const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);+ const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);+ const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);+ q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;+ const __m128i q2_1 = _mm_and_si128(q2bits, m3);+ const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);+ const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);+ const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);++ // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8+ __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);+ __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);+ __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);+ __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);+ __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);+ __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);+ __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);+ __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);++ // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8+ __m128i shuffle = _mm_set1_epi16(0x0100);+ p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);+ shuffle = _mm_add_epi16(shuffle, m2);+ p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);+ shuffle = _mm_add_epi16(shuffle, m2);+ p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);+ shuffle = _mm_add_epi16(shuffle, m2);+ p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);+ shuffle = _mm_add_epi16(shuffle, m2);+ p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);+ shuffle = _mm_add_epi16(shuffle, m2);+ p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);+ shuffle = _mm_add_epi16(shuffle, m2);+ p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);+ shuffle = _mm_add_epi16(shuffle, m2);+ p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);++ p0 = _mm_add_epi32(p0, p1);+ p2 = _mm_add_epi32(p2, p3);+ p4 = _mm_add_epi32(p4, p5);+ p6 = _mm_add_epi32(p6, p7);++ // isum in 32bits*4*2+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));+ }++ // sumf += dall * isum - dmin * summs in 32bits+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);+ }++ *s = hsum_float_8(acc);++#elif defined __riscv_v_intrinsic++ float sumf = 0;+ uint8_t temp_01[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};++ for (int i = 0; i < nb; ++i) {++ const uint8_t * q2 = x[i].qs;+ const int8_t * q8 = y[i].qs;+ const uint8_t * sc = x[i].scales;++ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ size_t vl = 16;++ vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl);+ vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl);++ vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl);++ vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl);+ vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl);+ vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));+ vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl);+ vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);++ sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums);++ vl = 32;++ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);+ vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl);++ uint8_t is=0;+ int isum=0;++ for (int j = 0; j < QK_K/128; ++j) {+ // load Q2+ vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl);++ vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl);+ vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03 , vl);+ vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03 , vl);+ vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03 , vl);++ // duplicate scale elements for product+ vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0+is, vl), vl);+ vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2+is, vl), vl);+ vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4+is, vl), vl);+ vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6+is, vl), vl);++ vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl));+ vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl));+ vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl));+ vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl));++ // load Q8+ vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);+ vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);+ vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8+64, vl);+ vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8+96, vl);++ vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl);+ vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl);+ vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl);+ vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl);++ vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl);+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl);++ isum += __riscv_vmv_x_s_i32m1_i32(isum1);++ q2+=32; q8+=128; is=8;++ }++ sumf += dall * isum;++ }++ *s = sumf;++#else++ float sumf = 0;++ for (int i = 0; i < nb; ++i) {++ const uint8_t * q2 = x[i].qs;+ const int8_t * q8 = y[i].qs;+ const uint8_t * sc = x[i].scales;++ int summs = 0;+ for (int j = 0; j < 16; ++j) {+ summs += y[i].bsums[j] * (sc[j] >> 4);+ }++ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ int isum = 0;+ int is = 0;+ int d;+ for (int k = 0; k < QK_K/128; ++k) {+ int shift = 0;+ for (int j = 0; j < 4; ++j) {+ d = sc[is++] & 0xF;+ int isuml = 0;+ for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);+ isum += d * isuml;+ d = sc[is++] & 0xF;+ isuml = 0;+ for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);+ isum += d * isuml;+ shift += 2;+ q8 += 32;+ }+ q2 += 32;+ }+ sumf += dall * isum - dmin * summs;+ }+ *s = sumf;+#endif+}++#else++void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_K * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#ifdef __ARM_NEON+ const uint8x16_t m3 = vdupq_n_u8(0x3);++ const int32x4_t vzero = vdupq_n_s32(0);++ ggml_int8x16x4_t q2bytes;++ uint32_t aux32[2];+ const uint8_t * scales = (const uint8_t *)aux32;++ float sum = 0;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;++ aux32[0] = sc[0] & 0x0f0f0f0f;+ aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;++ sum += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);++ int isum1 = 0, isum2 = 0;++ const uint8x16_t q2bits = vld1q_u8(q2);++ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);++ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits, m3));+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 2), m3));+ q2bytes.val[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 4), m3));+ q2bytes.val[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 6), m3));++ isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * scales[0];+ isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * scales[1];+ isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[2], q8bytes.val[2])) * scales[2];+ isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[3], q8bytes.val[3])) * scales[3];++ sum += d * (isum1 + isum2);+ }++ *s = sum;++#elif defined __AVX2__++ const __m256i m3 = _mm256_set1_epi8(3);++ __m256 acc = _mm256_setzero_ps();++ uint32_t ud, um;+ const uint8_t * restrict db = (const uint8_t *)&ud;+ const uint8_t * restrict mb = (const uint8_t *)&um;++ float summs = 0;++ // TODO: optimize this++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;+ ud = (sc[0] >> 0) & 0x0f0f0f0f;+ um = (sc[0] >> 4) & 0x0f0f0f0f;++ int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];+ summs += dmin * smin;++ const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);+ const __m256i q2_0 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 2), q2bits), m3);+ const __m256i q2_1 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 6), _mm_srli_epi16(q2bits, 4)), m3);++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));++ const __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);+ const __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);++ const __m256i p_0 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 0));+ const __m256i p_1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 1));+ const __m256i p_2 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 0));+ const __m256i p_3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 1));++ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0), acc);+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1), acc);+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2), acc);+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3), acc);+ }++ *s = hsum_float_8(acc) + summs;++#elif defined __AVX__++ const __m128i m3 = _mm_set1_epi8(3);++ __m256 acc = _mm256_setzero_ps();++ uint32_t ud, um;+ const uint8_t * restrict db = (const uint8_t *)&ud;+ const uint8_t * restrict mb = (const uint8_t *)&um;++ float summs = 0;++ // TODO: optimize this++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;+ ud = (sc[0] >> 0) & 0x0f0f0f0f;+ um = (sc[0] >> 4) & 0x0f0f0f0f;++ int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];+ summs += dmin * smin;++ const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);+ const __m128i q2_0 = _mm_and_si128(q2bits, m3);+ const __m128i q2_1 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);+ const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);+ const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));++ const __m128i p0 = _mm_maddubs_epi16(q2_0, _mm256_extractf128_si256(q8_0, 0));+ const __m128i p1 = _mm_maddubs_epi16(q2_1, _mm256_extractf128_si256(q8_0, 1));+ const __m128i p2 = _mm_maddubs_epi16(q2_2, _mm256_extractf128_si256(q8_1, 0));+ const __m128i p3 = _mm_maddubs_epi16(q2_3, _mm256_extractf128_si256(q8_1, 1));++ const __m256i p_0 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p0, p0)), _mm_cvtepi16_epi32(p0));+ const __m256i p_1 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p1, p1)), _mm_cvtepi16_epi32(p1));+ const __m256i p_2 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p2, p2)), _mm_cvtepi16_epi32(p2));+ const __m256i p_3 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p3, p3)), _mm_cvtepi16_epi32(p3));++ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0)), acc);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1)), acc);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2)), acc);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3)), acc);+ }++ *s = hsum_float_8(acc) + summs;++#elif defined __riscv_v_intrinsic++ uint32_t aux32[2];+ const uint8_t * scales = (const uint8_t *)aux32;++ float sumf = 0;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;++ aux32[0] = sc[0] & 0x0f0f0f0f;+ aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;++ sumf += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);++ int isum1 = 0;+ int isum2 = 0;++ size_t vl = 16;++ vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);++ // load Q2+ vuint8mf2_t q2_x = __riscv_vle8_v_u8mf2(q2, vl);++ vint8mf2_t q2_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q2_x, 0x03, vl));+ vint8mf2_t q2_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x2, vl), 0x03 , vl));+ vint8mf2_t q2_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x4, vl), 0x03 , vl));+ vint8mf2_t q2_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x6, vl), 0x03 , vl));++ // load Q8, and take product with Q2+ vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q2_0, __riscv_vle8_v_i8mf2(q8, vl), vl);+ vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q2_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);+ vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q2_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);+ vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q2_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);++ vint16m1_t vs_0 = __riscv_vredsum_vs_i16m1_i16m1(p0, vzero, vl);+ vint16m1_t vs_1 = __riscv_vredsum_vs_i16m1_i16m1(p1, vzero, vl);+ vint16m1_t vs_2 = __riscv_vredsum_vs_i16m1_i16m1(p2, vzero, vl);+ vint16m1_t vs_3 = __riscv_vredsum_vs_i16m1_i16m1(p3, vzero, vl);++ isum1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[0];+ isum2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[1];+ isum1 += __riscv_vmv_x_s_i16m1_i16(vs_2) * scales[2];+ isum2 += __riscv_vmv_x_s_i16m1_i16(vs_3) * scales[3];++ sumf += d * (isum1 + isum2);++ }++ *s = sumf;++#else++ float sumf = 0;++ int isum[QK_K/16];++ for (int i = 0; i < nb; ++i) {++ const uint8_t * q2 = x[i].qs;+ const int8_t * q8 = y[i].qs;+ const uint8_t * sc = x[i].scales;++ int summs = 0;+ for (int j = 0; j < QK_K/16; ++j) {+ summs += y[i].bsums[j] * (sc[j] >> 4);+ }++ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ memset(isum, 0, (QK_K/16)*sizeof(int));+ for (int l = 0; l < 16; ++l) {+ isum[0] += q8[l+ 0] * ((q2[l] >> 0) & 3);+ isum[1] += q8[l+16] * ((q2[l] >> 2) & 3);+ isum[2] += q8[l+32] * ((q2[l] >> 4) & 3);+ isum[3] += q8[l+48] * ((q2[l] >> 6) & 3);+ }+ for (int l = 0; l < QK_K/16; ++l) {+ isum[l] *= (sc[l] & 0xF);+ }+ sumf += dall * (isum[0] + isum[1] + isum[2] + isum[3]) - dmin * summs;+ }+ *s = sumf;+#endif+}+#endif++#if QK_K == 256+void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const uint32_t kmask1 = 0x03030303;+ const uint32_t kmask2 = 0x0f0f0f0f;++ const block_q3_K * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#ifdef __ARM_NEON++ uint32_t aux[3];+ uint32_t utmp[4];++ const uint8x16_t m3b = vdupq_n_u8(0x3);+ const int32x4_t vzero = vdupq_n_s32(0);++ const uint8x16_t m0 = vdupq_n_u8(1);+ const uint8x16_t m1 = vshlq_n_u8(m0, 1);+ const uint8x16_t m2 = vshlq_n_u8(m0, 2);+ const uint8x16_t m3 = vshlq_n_u8(m0, 3);+ const int8_t m32 = 32;++ ggml_int8x16x4_t q3bytes;++ float sum = 0;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict qh = x[i].hmask;+ const int8_t * restrict q8 = y[i].qs;++ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);++ ggml_uint8x16x4_t q3h;++ int32_t isum = 0;++ // Set up scales+ memcpy(aux, x[i].scales, 12);+ utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);+ utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);+ utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);+ utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);++ int8_t * scale = (int8_t *)utmp;+ for (int j = 0; j < 16; ++j) scale[j] -= m32;++ for (int j = 0; j < QK_K/128; ++j) {++ const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;+ const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;+ const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;++ q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);+ q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);+ q3h.val[2] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[0]), 1);+ q3h.val[3] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[1]), 1);++ q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[0], m3b)), vreinterpretq_s8_u8(q3h.val[0]));+ q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[1], m3b)), vreinterpretq_s8_u8(q3h.val[1]));+ q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));+ q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));++ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];++ scale += 4;++ q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);+ q3h.val[1] = vbicq_u8(m2, qhbits.val[1]);+ q3h.val[2] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[0]), 1);+ q3h.val[3] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[1]), 1);++ q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 4), m3b)), vreinterpretq_s8_u8(q3h.val[0]));+ q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 4), m3b)), vreinterpretq_s8_u8(q3h.val[1]));+ q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));+ q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));++ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];++ scale += 4;++ if (j == 0) {+ qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);+ qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);+ }++ }+ sum += d * isum;++ }++ *s = sum;++#elif defined __AVX2__++ const __m256i m3 = _mm256_set1_epi8(3);+ const __m256i mone = _mm256_set1_epi8(1);+ const __m128i m32 = _mm_set1_epi8(32);++ __m256 acc = _mm256_setzero_ps();++ uint32_t aux[3];++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ // Set up scales+ memcpy(aux, x[i].scales, 12);+ __m128i scales128 = _mm_set_epi32(+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));+ scales128 = _mm_sub_epi8(scales128, m32);+ const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};++ // high bit+ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);++ // integer accumulator+ __m256i sumi = _mm256_setzero_si256();++ int bit = 0;+ int is = 0;++ for (int j = 0; j < QK_K/128; ++j) {+ // load low 2 bits+ const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;++ // prepare low and high bits+ const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);+ const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);+ const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);+ const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);+ const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);+ ++bit;++ // load Q8 quants+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);+ __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);+ __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);+ __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);++ __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);+ __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);+ __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);++ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);++ // multiply with scales+ p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);+ p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);+ p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);+ p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);++ // accumulate+ p16_0 = _mm256_add_epi32(p16_0, p16_1);+ p16_2 = _mm256_add_epi32(p16_2, p16_3);+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));++ }++ // multiply with block scale and accumulate+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);++ }++ *s = hsum_float_8(acc);++#elif defined __AVX__++ const __m128i m3 = _mm_set1_epi8(3);+ const __m128i mone = _mm_set1_epi8(1);+ const __m128i m32 = _mm_set1_epi8(32);+ const __m128i m2 = _mm_set1_epi8(2);++ __m256 acc = _mm256_setzero_ps();++ const uint32_t *aux;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ // Set up scales+ aux = (const uint32_t *)x[i].scales;+ __m128i scales128 = _mm_set_epi32(+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));+ scales128 = _mm_sub_epi8(scales128, m32);+ const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));+ const __m128i scales[2] = { scales_0, scales_1 };++ // high bit *128*2 from block_q3_K.hmask[QK_K/8]+ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);++ // integer accumulator+ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ for (int j = 0; j < QK_K/128; ++j) {+ // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]+ const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;+ const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;++ // prepare low and high bits+ const int bit = j << 2;++ const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);+ const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);+ const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);+ const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);++ const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);+ const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);+ const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);+ const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);++ const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);+ const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);+ const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);+ const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);++ const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);+ const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);+ const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);+ const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);++ // load Q8 quants from block_q8_K.qs[QK_K]+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);+ __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);+ __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);+ __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);+ __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);+ __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);+ __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);+ __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);++ __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);+ __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);+ __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);+ __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);+ __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);+ __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);+ __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);+ __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);++ p16_0 = _mm_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);++ // multiply with scales+ __m128i shuffle = _mm_set1_epi16(0x0100);+ p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);+ shuffle = _mm_add_epi16(shuffle, m2);+ p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);++ // accumulate+ p16_0 = _mm_add_epi32(p16_0, p16_1);+ p16_2 = _mm_add_epi32(p16_2, p16_3);+ p16_4 = _mm_add_epi32(p16_4, p16_5);+ p16_6 = _mm_add_epi32(p16_6, p16_7);+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));++ }++ // multiply with block scale and accumulate+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);++ }++ *s = hsum_float_8(acc);++#elif defined __riscv_v_intrinsic++ uint32_t aux[3];+ uint32_t utmp[4];++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {++ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict qh = x[i].hmask;+ const int8_t * restrict q8 = y[i].qs;++ memcpy(aux, x[i].scales, 12);+ utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);+ utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);+ utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);+ utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);++ int8_t * scale = (int8_t *)utmp;+ for (int j = 0; j < 16; ++j) scale[j] -= 32;+++ size_t vl = 32;+ uint8_t m = 1;++ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);+ vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl);++ int sum_t = 0;++ for (int j = 0; j < QK_K; j += 128) {++ vl = 32;++ // load Q3+ vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl);++ vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl));+ vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl));+ vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl));+ vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl));++ // compute mask for subtraction+ vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl);+ vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl);+ vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_m(vmask_0, q3_0, 0x4, vl);+ m <<= 1;++ vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);+ vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl);+ vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_m(vmask_1, q3_1, 0x4, vl);+ m <<= 1;++ vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);+ vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl);+ vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_m(vmask_2, q3_2, 0x4, vl);+ m <<= 1;++ vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl);+ vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl);+ vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_m(vmask_3, q3_3, 0x4, vl);+ m <<= 1;++ // load Q8 and take product with Q3+ vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl);+ vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl);+ vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl);+ vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl);++ vl = 16;++ // retrieve lane to multiply with scale+ vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl);+ vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl);+ vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl);+ vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl);+ vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl);+ vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl);+ vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl);+ vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl);++ vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl);+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl);+ vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl);+ vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl);++ sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);++ q3 += 32; q8 += 128; scale += 8;++ }++ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;++ sumf += d*sum_t;++ }++ *s = sumf;++#else+ // scalar version+ // This function is written like this so the compiler can manage to vectorize most of it+ // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the+ // manually vectorized version above. Every other version I tried would run at least 4 times slower.+ // The ideal situation would be if we could just write the code once, and the compiler would+ // automatically produce the best possible set of machine instructions, instead of us having to manually+ // write vectorized versions for AVX, ARM_NEON, etc.++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ uint32_t auxs[4];+ const int8_t * scales = (const int8_t*)auxs;++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict hm = x[i].hmask;+ const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * restrict a = aux8;+ uint8_t m = 1;+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);+ a += 32; m <<= 1;+ q3 += 32;+ }+ a = aux8;++ memcpy(auxs, x[i].scales, 12);+ uint32_t tmp = auxs[2];+ auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);+ auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);+ auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);+ auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);+ for (int j = 0; j < QK_K/16; ++j) {+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;++#endif++}++#else++void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q3_K * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#ifdef __ARM_NEON+ const int32x4_t vzero = vdupq_n_s32(0);++ const uint8x16_t m3b = vdupq_n_u8(0x3);+ const uint8x16_t mh = vdupq_n_u8(4);++ ggml_int8x16x4_t q3bytes;++ uint16_t aux16[2];+ int8_t * scales = (int8_t *)aux16;++ float sum = 0;++ for (int i = 0; i < nb; ++i) {++ ggml_uint8x16x4_t q3h;++ const uint8x8_t hbits = vld1_u8(x[i].hmask);+ const uint8x16_t q3bits = vld1q_u8(x[i].qs);+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(y[i].qs);++ const uint16_t a = *(const uint16_t *)x[i].scales;+ aux16[0] = a & 0x0f0f;+ aux16[1] = (a >> 4) & 0x0f0f;++ for (int j = 0; j < 4; ++j) scales[j] -= 8;++ int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8x16_t htmp = vcombine_u8(hbits, vshr_n_u8(hbits, 1));+ q3h.val[0] = vandq_u8(mh, vshlq_n_u8(htmp, 2));+ q3h.val[1] = vandq_u8(mh, htmp);+ q3h.val[2] = vandq_u8(mh, vshrq_n_u8(htmp, 2));+ q3h.val[3] = vandq_u8(mh, vshrq_n_u8(htmp, 4));++ q3bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q3bits, m3b), q3h.val[0]));+ q3bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 2), m3b), q3h.val[1]));+ q3bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 4), m3b), q3h.val[2]));+ q3bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q3bits, 6), q3h.val[3]));++ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes.val[0])) * scales[0];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes.val[1])) * scales[2];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes.val[2])) * scales[1];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes.val[3])) * scales[3];++ sum += d * isum;++ }++ *s = sum;++#elif defined __AVX2__++ const __m256i m3 = _mm256_set1_epi8(3);+ const __m256i m1 = _mm256_set1_epi8(1);++ __m256 acc = _mm256_setzero_ps();++ uint64_t aux64;++ uint16_t aux16[2];+ const int8_t * aux8 = (const int8_t *)aux16;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const uint16_t a = *(const uint16_t *)x[i].scales;+ aux16[0] = a & 0x0f0f;+ aux16[1] = (a >> 4) & 0x0f0f;++ const __m256i scale_0 = MM256_SET_M128I(_mm_set1_epi16(aux8[2] - 8), _mm_set1_epi16(aux8[0] - 8));+ const __m256i scale_1 = MM256_SET_M128I(_mm_set1_epi16(aux8[3] - 8), _mm_set1_epi16(aux8[1] - 8));++ memcpy(&aux64, x[i].hmask, 8);++ const __m128i haux = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);+ __m256i q3h_0 = MM256_SET_M128I(_mm_srli_epi16(haux, 2), haux);+ __m256i q3h_1 = _mm256_srli_epi16(q3h_0, 4);+ q3h_0 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_0, m1), 2);+ q3h_1 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_1, m1), 2);++ // load low 2 bits+ const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);++ // prepare low and high bits+ const __m256i q3aux = MM256_SET_M128I(_mm_srli_epi16(q3bits, 2), q3bits);+ const __m256i q3l_0 = _mm256_and_si256(q3aux, m3);+ const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3aux, 4), m3);++ // load Q8 quants+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ const __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);+ const __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);++ __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);++ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);++ // multiply with scales+ p16_0 = _mm256_madd_epi16(scale_0, p16_0);+ p16_1 = _mm256_madd_epi16(scale_1, p16_1);++ p16_0 = _mm256_add_epi32(p16_0, p16_1);++ // multiply with block scale and accumulate+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16_0), acc);++ }++ *s = hsum_float_8(acc);++#elif defined __AVX__++ const __m128i m3 = _mm_set1_epi8(3);+ const __m128i m1 = _mm_set1_epi8(1);++ __m256 acc = _mm256_setzero_ps();++ uint64_t aux64;++ uint16_t aux16[2];+ const int8_t * aux8 = (const int8_t *)aux16;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const uint16_t a = *(const uint16_t *)x[i].scales;+ aux16[0] = a & 0x0f0f;+ aux16[1] = (a >> 4) & 0x0f0f;++ const __m128i scale_0 = _mm_set1_epi16(aux8[0] - 8);+ const __m128i scale_1 = _mm_set1_epi16(aux8[2] - 8);+ const __m128i scale_2 = _mm_set1_epi16(aux8[1] - 8);+ const __m128i scale_3 = _mm_set1_epi16(aux8[3] - 8);++ memcpy(&aux64, x[i].hmask, 8);++ __m128i q3h_0 = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);+ __m128i q3h_1 = _mm_srli_epi16(q3h_0, 2);+ __m128i q3h_2 = _mm_srli_epi16(q3h_0, 4);+ __m128i q3h_3 = _mm_srli_epi16(q3h_0, 6);+ q3h_0 = _mm_slli_epi16(_mm_andnot_si128(q3h_0, m1), 2);+ q3h_1 = _mm_slli_epi16(_mm_andnot_si128(q3h_1, m1), 2);+ q3h_2 = _mm_slli_epi16(_mm_andnot_si128(q3h_2, m1), 2);+ q3h_3 = _mm_slli_epi16(_mm_andnot_si128(q3h_3, m1), 2);++ // load low 2 bits+ const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);++ // prepare low and high bits+ const __m128i q3l_0 = _mm_and_si128(q3bits, m3);+ const __m128i q3l_1 = _mm_and_si128(_mm_srli_epi16(q3bits, 2), m3);+ const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits, 4), m3);+ const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits, 6), m3);++ // load Q8 quants+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm_maddubs_epi16,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ const __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, _mm256_extractf128_si256(q8_0, 0));+ const __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, _mm256_extractf128_si256(q8_0, 1));+ const __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, _mm256_extractf128_si256(q8_1, 0));+ const __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, _mm256_extractf128_si256(q8_1, 1));++ __m128i p16_0 = _mm_maddubs_epi16(q3l_0, _mm256_extractf128_si256(q8_0, 0));+ __m128i p16_1 = _mm_maddubs_epi16(q3l_1, _mm256_extractf128_si256(q8_0, 1));+ __m128i p16_2 = _mm_maddubs_epi16(q3l_2, _mm256_extractf128_si256(q8_1, 0));+ __m128i p16_3 = _mm_maddubs_epi16(q3l_3, _mm256_extractf128_si256(q8_1, 1));++ p16_0 = _mm_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);++ // multiply with scales+ p16_0 = _mm_madd_epi16(scale_0, p16_0);+ p16_1 = _mm_madd_epi16(scale_1, p16_1);+ p16_2 = _mm_madd_epi16(scale_2, p16_2);+ p16_3 = _mm_madd_epi16(scale_3, p16_3);++ p16_0 = _mm_add_epi32(p16_0, p16_2);+ p16_1 = _mm_add_epi32(p16_1, p16_3);+ __m256i p16 = MM256_SET_M128I(p16_1, p16_0);++ // multiply with block scale and accumulate+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16)), acc);++ }++ *s = hsum_float_8(acc);++#elif defined __riscv_v_intrinsic++ uint16_t aux16[2];+ int8_t * scales = (int8_t *)aux16;++ float sumf = 0;++ for (int i = 0; i < nb; ++i) {++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const uint16_t a = *(const uint16_t *)x[i].scales;+ aux16[0] = a & 0x0f0f;+ aux16[1] = (a >> 4) & 0x0f0f;++ for (int j = 0; j < 4; ++j) scales[j] -= 8;++ int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);++ // load qh+ vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(x[i].hmask, 8);+ vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));++ size_t vl = 16;++ // extend and combine both qh_x1 and qh_x2+ vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);++ vuint8mf2_t qh_0 = __riscv_vand_vx_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);+ vuint8mf2_t qh_1 = __riscv_vand_vx_u8mf2(qh_x, 0x4, vl);+ vuint8mf2_t qh_2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);+ vuint8mf2_t qh_3 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), 0x4, vl);++ // load Q3+ vuint8mf2_t q3_x = __riscv_vle8_v_u8mf2(q3, vl);++ vuint8mf2_t q3h_0 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(q3_x, 0x3, vl), qh_0, vl);+ vuint8mf2_t q3h_1 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 2, vl), 0x3, vl), qh_1, vl);+ vuint8mf2_t q3h_2 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 4, vl), 0x3, vl), qh_2, vl);+ vuint8mf2_t q3h_3 = __riscv_vor_vv_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x6, vl), qh_3, vl);++ vint8mf2_t q3_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_0);+ vint8mf2_t q3_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_1);+ vint8mf2_t q3_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_2);+ vint8mf2_t q3_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_3);++ // load Q8 and take product with Q3+ vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q3_0, __riscv_vle8_v_i8mf2(q8, vl), vl);+ vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q3_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);+ vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q3_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);+ vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q3_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);++ vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);+ vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);+ vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);+ vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);++ isum += __riscv_vmv_x_s_i32m1_i32(vs_0) * scales[0];+ isum += __riscv_vmv_x_s_i32m1_i32(vs_1) * scales[2];+ isum += __riscv_vmv_x_s_i32m1_i32(vs_2) * scales[1];+ isum += __riscv_vmv_x_s_i32m1_i32(vs_3) * scales[3];++ sumf += d * isum;++ }++ *s = sumf;++#else++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ int32_t scales[4];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict hm = x[i].hmask;+ const int8_t * restrict q8 = y[i].qs;+ int8_t * restrict a = aux8;+ for (int l = 0; l < 8; ++l) {+ a[l+ 0] = (int8_t)((q3[l+0] >> 0) & 3) - (hm[l] & 0x01 ? 0 : 4);+ a[l+ 8] = (int8_t)((q3[l+8] >> 0) & 3) - (hm[l] & 0x02 ? 0 : 4);+ a[l+16] = (int8_t)((q3[l+0] >> 2) & 3) - (hm[l] & 0x04 ? 0 : 4);+ a[l+24] = (int8_t)((q3[l+8] >> 2) & 3) - (hm[l] & 0x08 ? 0 : 4);+ a[l+32] = (int8_t)((q3[l+0] >> 4) & 3) - (hm[l] & 0x10 ? 0 : 4);+ a[l+40] = (int8_t)((q3[l+8] >> 4) & 3) - (hm[l] & 0x20 ? 0 : 4);+ a[l+48] = (int8_t)((q3[l+0] >> 6) & 3) - (hm[l] & 0x40 ? 0 : 4);+ a[l+56] = (int8_t)((q3[l+8] >> 6) & 3) - (hm[l] & 0x80 ? 0 : 4);+ }++ scales[0] = (x[i].scales[0] & 0xF) - 8;+ scales[1] = (x[i].scales[0] >> 4) - 8;+ scales[2] = (x[i].scales[1] & 0xF) - 8;+ scales[3] = (x[i].scales[1] >> 4) - 8;++ memset(aux32, 0, 8*sizeof(int32_t));+ for (int j = 0; j < QK_K/16; ++j) {+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] += q8[l] * a[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux32[l] += scales[j] * aux16[l];+ }+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;++#endif++}+#endif++#if QK_K == 256+void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_K * restrict x = vx;+ const block_q8_K * restrict y = vy;-// We use this macro instead of a function call because for some reason-// the code runs 2-3% slower, even if the function is declared inline-#define MULTIPLY_ACCUM_WITH_SCALE(index)\- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];+ const int nb = n / QK_K;-#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\- q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\- q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\- MULTIPLY_ACCUM_WITH_SCALE((index));+ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;- for (int j = 0; j < QK_K/128; ++j) {- const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;+ uint32_t utmp[4];- ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;- q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));- q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));+#ifdef __ARM_NEON+ const uint8x16_t m4b = vdupq_n_u8(0xf);+ const int32x4_t mzero = vdupq_n_s32(0);- MULTIPLY_ACCUM_WITH_SCALE(0);+ ggml_int8x16x2_t q4bytes;+ ggml_int8x16x2_t q8bytes;- SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);- SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);- SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);+ float sumf = 0;- is += 8;+ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));++ memcpy(utmp, x[i].scales, 12);++ uint32x2_t mins8 = { 0 };+ mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);+ mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);++ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[0] &= kmask1;++ const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));+ const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),+ vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));+ sumf -= dmin * vaddvq_s32(prod);++ const uint8_t * scales = (const uint8_t *)utmp;++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ int32_t sumi1 = 0;+ int32_t sumi2 = 0;++ for (int j = 0; j < QK_K/64; ++j) {+ const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;++ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;+ q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));+ q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));++ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);+ sumi1 += vaddvq_s32(p1) * scales[2*j+0];++ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;+ q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));+ q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));++ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);++ sumi2 += vaddvq_s32(p2) * scales[2*j+1];}- sum += d * isum;+ sumf += d * (sumi1 + sumi2);+}- *s = sum;+ *s = sumf;#elif defined __AVX2__- const __m256i m3 = _mm256_set1_epi8(3);- const __m128i m4 = _mm_set1_epi8(0xF);+ const __m256i m4 = _mm256_set1_epi8(0xF);__m256 acc = _mm256_setzero_ps();+ __m128 acc_m = _mm_setzero_ps();- for (int i = 0; i < nb; ++i) {+ for (int i = 0; i < nb; ++i) {const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);- const uint8_t * restrict q2 = x[i].qs;+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ const uint8_t * restrict q4 = x[i].qs;const int8_t * restrict q8 = y[i].qs;- const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);- const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);- const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);- const __m256i mins = _mm256_cvtepi8_epi16(mins8);- const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));+ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);+ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);+ acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);- const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);- const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);- const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);- const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};+ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);+ const __m256i scales = MM256_SET_M128I(sc128, sc128);__m256i sumi = _mm256_setzero_si256();- for (int j = 0; j < QK_K/128; ++j) {-- const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;-- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ for (int j = 0; j < QK_K/64; ++j) {- const __m256i q2_0 = _mm256_and_si256(q2bits, m3);- const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);- const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);- const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);+ const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));+ const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));- __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);- __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);- __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);- __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);+ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4l = _mm256_and_si256(q4bits, m4);+ const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);- p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);- p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);- p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);- p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);+ const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);+ p16l = _mm256_madd_epi16(scale_l, p16l);- p0 = _mm256_add_epi32(p0, p1);- p2 = _mm256_add_epi32(p2, p3);+ const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);+ p16h = _mm256_madd_epi16(scale_h, p16h);+ const __m256i sumj = _mm256_add_epi32(p16l, p16h);- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));+ sumi = _mm256_add_epi32(sumi, sumj);}- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);+ __m256 vd = _mm256_set1_ps(d);+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);}- *s = hsum_float_8(acc);+ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));++ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);#elif defined __AVX__- const __m128i m3 = _mm_set1_epi8(0x3);const __m128i m4 = _mm_set1_epi8(0xF);const __m128i m2 = _mm_set1_epi8(0x2);__m256 acc = _mm256_setzero_ps();+ __m128 acc_m = _mm_setzero_ps();- for (int i = 0; i < nb; ++i) {+ for (int i = 0; i < nb; ++i) {- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);- const uint8_t * restrict q2 = x[i].qs;+ const uint8_t * restrict q4 = x[i].qs;const int8_t * restrict q8 = y[i].qs;- // load mins and scales from block_q2_K.scales[QK_K/16]- const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);- const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);- const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);- const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);- const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));-- // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2- const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));- const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;- // sumf += -dmin * summs in 32bits*8- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);+ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);+ const __m128i scales = _mm_cvtepu8_epi16(utmps);+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));- const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);- const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));- const __m128i scales[2] = { scales_0, scales_1 };+ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);+ const __m128i prod = _mm_madd_epi16(mins, q8s);+ acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);__m128i sumi_0 = _mm_setzero_si128();__m128i sumi_1 = _mm_setzero_si128();- for (int j = 0; j < QK_K/128; ++j) {-- // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]- const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;-- // load 2bits*16*8 from block_q2_K.qs[QK_K/4]- __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;- const __m128i q2_0 = _mm_and_si128(q2bits, m3);- const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);- const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);- const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);- q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;- const __m128i q2_1 = _mm_and_si128(q2bits, m3);- const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);- const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);- const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);-- // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8- __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);- __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);- __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);- __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);- __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);- __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);- __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);- __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);+ __m128i shuffle = _mm_set1_epi16(0x0100);+ for (int j = 0; j < QK_K/64; ++j) {- // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8- __m128i shuffle = _mm_set1_epi16(0x0100);- p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);- shuffle = _mm_add_epi16(shuffle, m2);- p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);- shuffle = _mm_add_epi16(shuffle, m2);- p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);- shuffle = _mm_add_epi16(shuffle, m2);- p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);- shuffle = _mm_add_epi16(shuffle, m2);- p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);- shuffle = _mm_add_epi16(shuffle, m2);- p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);+ const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);shuffle = _mm_add_epi16(shuffle, m2);- p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);+ const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);shuffle = _mm_add_epi16(shuffle, m2);- p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);-- p0 = _mm_add_epi32(p0, p1);- p2 = _mm_add_epi32(p2, p3);- p4 = _mm_add_epi32(p4, p5);- p6 = _mm_add_epi32(p6, p7);- // isum in 32bits*4*2- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));- }-- // sumf += dall * isum - dmin * summs in 32bits+ __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4l_0 = _mm_and_si128(q4bits, m4);+ const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);+ q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4l_1 = _mm_and_si128(q4bits, m4);+ const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);++ const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);+ p16l = _mm_madd_epi16(scale_l, p16l);+ sumi_0 = _mm_add_epi32(sumi_0, p16l);+ const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ p16l = _mm_maddubs_epi16(q4l_1, q8l_1);+ p16l = _mm_madd_epi16(scale_l, p16l);+ sumi_1 = _mm_add_epi32(sumi_1, p16l);++ const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);+ p16h = _mm_madd_epi16(scale_h, p16h);+ sumi_0 = _mm_add_epi32(sumi_0, p16h);+ const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ p16h = _mm_maddubs_epi16(q4h_1, q8h_1);+ p16h = _mm_madd_epi16(scale_h, p16h);+ sumi_1 = _mm_add_epi32(sumi_1, p16h);++ }++ __m256 vd = _mm256_set1_ps(d);__m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);+}- *s = hsum_float_8(acc);+ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));++ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);#elif defined __riscv_v_intrinsic+ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];+float sumf = 0;- uint8_t temp_01[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};for (int i = 0; i < nb; ++i) {- const uint8_t * q2 = x[i].qs;- const int8_t * q8 = y[i].qs;- const uint8_t * sc = x[i].scales;+ size_t vl = 8;- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);- size_t vl = 16;+ vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);+ vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);+ vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);- vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl);- vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl);+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;- vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl);+ vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);+ vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));+ vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);- vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl);- vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl);- vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));- vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl);- vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);+ vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);+ sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);- sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums);+ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;vl = 32;- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);- vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl);-- uint8_t is=0;- int isum=0;-- for (int j = 0; j < QK_K/128; ++j) {- // load Q2- vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl);-- vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl);- vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03 , vl);- vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03 , vl);- vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03 , vl);+ int32_t sum_1 = 0;+ int32_t sum_2 = 0;- // duplicate scale elements for product- vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0+is, vl), vl);- vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2+is, vl), vl);- vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4+is, vl), vl);- vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6+is, vl), vl);+ vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);- vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl));- vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl));- vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl));- vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl));+ for (int j = 0; j < QK_K/64; ++j) {+ // load Q4+ vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);- // load Q8- vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);- vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);- vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8+64, vl);- vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8+96, vl);+ // load Q8 and multiply it with lower Q4 nibble+ vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);+ vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));+ vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);+ vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);- vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl);- vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl);- vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl);- vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl);+ sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];- vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl);- vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl);+ // load Q8 and multiply it with upper Q4 nibble+ vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);+ vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));+ vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);+ vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);- isum += __riscv_vmv_x_s_i32m1_i32(isum1);+ sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];- q2+=32; q8+=128; is=8;+ q4 += 32; q8 += 64;}- sumf += dall * isum;+ sumf += d*(sum_1 + sum_2);}@@ -4313,269 +6557,265 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r#else- float sumf = 0;-- for (int i = 0; i < nb; ++i) {- const uint8_t * q2 = x[i].qs;- const int8_t * q8 = y[i].qs;- const uint8_t * sc = x[i].scales;+ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];- int summs = 0;- for (int j = 0; j < 16; ++j) {- summs += y[i].bsums[j] * (sc[j] >> 4);- }+ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * restrict a = aux8;+ for (int j = 0; j < QK_K/64; ++j) {+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);+ a += 32;+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);+ a += 32; q4 += 32;+ }+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;- int isum = 0;+ int sumi = 0;+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];+ a = aux8;int is = 0;- int d;- for (int k = 0; k < QK_K/128; ++k) {- int shift = 0;- for (int j = 0; j < 4; ++j) {- d = sc[is++] & 0xF;- int isuml = 0;- for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);- isum += d * isuml;- d = sc[is++] & 0xF;- isuml = 0;- for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);- isum += d * isuml;- shift += 2;- q8 += 32;- }- q2 += 32;+ for (int j = 0; j < QK_K/32; ++j) {+ int32_t scale = scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;}- sumf += dall * isum - dmin * summs;+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;+ sumf -= dmin * sumi;}+ for (int l = 0; l < 8; ++l) sumf += sums[l];*s = sumf;#endif}-#else--void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);assert(nrc == 1);UNUSED(nrc);UNUSED(bx);UNUSED(by);UNUSED(bs);- const block_q2_K * restrict x = vx;+ const block_q4_K * restrict x = vx;const block_q8_K * restrict y = vy;const int nb = n / QK_K;#ifdef __ARM_NEON- const uint8x16_t m3 = vdupq_n_u8(0x3);+ const uint8x16_t m4b = vdupq_n_u8(0xf);- const int32x4_t vzero = vdupq_n_s32(0);+ const int32x4_t mzero = vdupq_n_s32(0);- ggml_int8x16x4_t q2bytes;+ float sumf = 0;- uint32_t aux32[2];- const uint8_t * scales = (const uint8_t *)aux32;+ ggml_int8x16x2_t q4bytes;+ ggml_int8x16x4_t q8bytes;- float sum = 0;+ float sum_mins = 0.f;- for (int i = 0; i < nb; ++i) {+ uint16_t aux16[2];+ const uint8_t * restrict scales = (const uint8_t *)aux16;- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q2 = x[i].qs;+ const uint8_t * restrict q4 = x[i].qs;const int8_t * restrict q8 = y[i].qs;- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;- aux32[0] = sc[0] & 0x0f0f0f0f;- aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;+ const uint16_t * restrict a = (const uint16_t *)x[i].scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;- sum += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);+ const int32_t summi = scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]);+ sum_mins += y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * summi;- int isum1 = 0, isum2 = 0;+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);- const uint8x16_t q2bits = vld1q_u8(q2);+ const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4);- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);+ q8bytes = ggml_vld1q_s8_x4(q8);+ q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));+ q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));- q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits, m3));- q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 2), m3));- q2bytes.val[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 4), m3));- q2bytes.val[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 6), m3));+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);+ const int32_t sumi1 = vaddvq_s32(p1) * scales[0];- isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * scales[0];- isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * scales[1];- isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[2], q8bytes.val[2])) * scales[2];- isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[3], q8bytes.val[3])) * scales[3];+ q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));+ q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));- sum += d * (isum1 + isum2);+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[2]), q4bytes.val[1], q8bytes.val[3]);+ const int32_t sumi2 = vaddvq_s32(p2) * scales[1];++ sumf += d * (sumi1 + sumi2);}- *s = sum;+ *s = sumf - sum_mins;#elif defined __AVX2__- const __m256i m3 = _mm256_set1_epi8(3);+ const __m256i m4 = _mm256_set1_epi8(0xF);__m256 acc = _mm256_setzero_ps();- uint32_t ud, um;- const uint8_t * restrict db = (const uint8_t *)&ud;- const uint8_t * restrict mb = (const uint8_t *)&um;-float summs = 0;- // TODO: optimize this+ uint16_t aux16[2];+ const uint8_t * scales = (const uint8_t *)aux16;for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;+ const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;+ const __m256 vd = _mm256_set1_ps(d);- const uint8_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;- ud = (sc[0] >> 0) & 0x0f0f0f0f;- um = (sc[0] >> 4) & 0x0f0f0f0f;+ summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));- int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];- summs += dmin * smin;+ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;- const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);- const __m256i q2_0 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 2), q2bits), m3);- const __m256i q2_1 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 6), _mm_srli_epi16(q2bits, 4)), m3);+ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);+ const __m256i q4l = _mm256_and_si256(q4bits, m4);+ const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));+ const __m256i q8l = _mm256_loadu_si256((const __m256i*)(q8+ 0));+ const __m256i q8h = _mm256_loadu_si256((const __m256i*)(q8+32));- const __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);- const __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);+ const __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);+ const __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);- const __m256i p_0 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 0));- const __m256i p_1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 1));- const __m256i p_2 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 0));- const __m256i p_3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 1));+ const __m256i p32l = _mm256_madd_epi16(_mm256_set1_epi16(scales[0]), p16l);+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32l), acc);++ const __m256i p32h = _mm256_madd_epi16(_mm256_set1_epi16(scales[1]), p16h);+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32h), acc);- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0), acc);- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1), acc);- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2), acc);- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3), acc);}- *s = hsum_float_8(acc) + summs;+ *s = hsum_float_8(acc) - summs;#elif defined __AVX__- const __m128i m3 = _mm_set1_epi8(3);+ const __m128i m4 = _mm_set1_epi8(0xF);__m256 acc = _mm256_setzero_ps();- uint32_t ud, um;- const uint8_t * restrict db = (const uint8_t *)&ud;- const uint8_t * restrict mb = (const uint8_t *)&um;-float summs = 0;- // TODO: optimize this+ uint16_t aux16[2];+ const uint8_t * scales = (const uint8_t *)aux16;for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;+ const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;+ const __m256 vd = _mm256_set1_ps(d);- const uint8_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;- ud = (sc[0] >> 0) & 0x0f0f0f0f;- um = (sc[0] >> 4) & 0x0f0f0f0f;+ summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));- int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];- summs += dmin * smin;+ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;- const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);- const __m128i q2_0 = _mm_and_si128(q2bits, m3);- const __m128i q2_1 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);- const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);- const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);+ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);+ const __m128i q4bits_0 = _mm256_extractf128_si256(q4bits, 0);+ const __m128i q4bits_1 = _mm256_extractf128_si256(q4bits, 1);+ const __m128i q4_0 = _mm_and_si128(q4bits_0, m4);+ const __m128i q4_1 = _mm_and_si128(q4bits_1, m4);+ const __m128i q4_2 = _mm_and_si128(_mm_srli_epi16(q4bits_0, 4), m4);+ const __m128i q4_3 = _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4);const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));- const __m128i p0 = _mm_maddubs_epi16(q2_0, _mm256_extractf128_si256(q8_0, 0));- const __m128i p1 = _mm_maddubs_epi16(q2_1, _mm256_extractf128_si256(q8_0, 1));- const __m128i p2 = _mm_maddubs_epi16(q2_2, _mm256_extractf128_si256(q8_1, 0));- const __m128i p3 = _mm_maddubs_epi16(q2_3, _mm256_extractf128_si256(q8_1, 1));+ const __m128i p16_0 = _mm_maddubs_epi16(q4_0, _mm256_extractf128_si256(q8_0, 0));+ const __m128i p16_1 = _mm_maddubs_epi16(q4_1, _mm256_extractf128_si256(q8_0, 1));+ const __m128i p16_2 = _mm_maddubs_epi16(q4_2, _mm256_extractf128_si256(q8_1, 0));+ const __m128i p16_3 = _mm_maddubs_epi16(q4_3, _mm256_extractf128_si256(q8_1, 1));- const __m256i p_0 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p0, p0)), _mm_cvtepi16_epi32(p0));- const __m256i p_1 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p1, p1)), _mm_cvtepi16_epi32(p1));- const __m256i p_2 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p2, p2)), _mm_cvtepi16_epi32(p2));- const __m256i p_3 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p3, p3)), _mm_cvtepi16_epi32(p3));+ const __m128i p32_0 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_0);+ const __m128i p32_1 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_1);+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_1, p32_0))), acc);++ const __m128i p32_2 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_2);+ const __m128i p32_3 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_3);+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_3, p32_2))), acc);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0)), acc);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1)), acc);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2)), acc);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3)), acc);}- *s = hsum_float_8(acc) + summs;+ *s = hsum_float_8(acc) - summs;#elif defined __riscv_v_intrinsic- uint32_t aux32[2];- const uint8_t * scales = (const uint8_t *)aux32;+ uint16_t s16[2];+ const uint8_t * restrict scales = (const uint8_t *)s16;float sumf = 0;for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- const uint8_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;-- aux32[0] = sc[0] & 0x0f0f0f0f;- aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;+ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;- sumf += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);+ const uint16_t * restrict b = (const uint16_t *)x[i].scales;+ s16[0] = b[0] & 0x0f0f;+ s16[1] = (b[0] >> 4) & 0x0f0f;- int isum1 = 0;- int isum2 = 0;+ sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);- size_t vl = 16;+ size_t vl = 32;vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);- // load Q2- vuint8mf2_t q2_x = __riscv_vle8_v_u8mf2(q2, vl);-- vint8mf2_t q2_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q2_x, 0x03, vl));- vint8mf2_t q2_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x2, vl), 0x03 , vl));- vint8mf2_t q2_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x4, vl), 0x03 , vl));- vint8mf2_t q2_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x6, vl), 0x03 , vl));-- // load Q8, and take product with Q2- vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q2_0, __riscv_vle8_v_i8mf2(q8, vl), vl);- vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q2_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);- vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q2_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);- vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q2_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);+ // load Q4+ vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);- vint16m1_t vs_0 = __riscv_vredsum_vs_i16m1_i16m1(p0, vzero, vl);- vint16m1_t vs_1 = __riscv_vredsum_vs_i16m1_i16m1(p1, vzero, vl);- vint16m1_t vs_2 = __riscv_vredsum_vs_i16m1_i16m1(p2, vzero, vl);- vint16m1_t vs_3 = __riscv_vredsum_vs_i16m1_i16m1(p3, vzero, vl);+ // load Q8 and multiply it with lower Q4 nibble+ vint8m1_t q4_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));+ vint16m2_t va_0 = __riscv_vwmul_vv_i16m2(q4_a, __riscv_vle8_v_i8m1(q8, vl), vl);+ vint16m1_t aux1 = __riscv_vredsum_vs_i16m2_i16m1(va_0, vzero, vl);- isum1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[0];- isum2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[1];- isum1 += __riscv_vmv_x_s_i16m1_i16(vs_2) * scales[2];- isum2 += __riscv_vmv_x_s_i16m1_i16(vs_3) * scales[3];+ sumf += d*scales[0]*__riscv_vmv_x_s_i16m1_i16(aux1);- sumf += d * (isum1 + isum2);+ // load Q8 and multiply it with upper Q4 nibble+ vint8m1_t q4_s = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));+ vint16m2_t va_1 = __riscv_vwmul_vv_i16m2(q4_s, __riscv_vle8_v_i8m1(q8+32, vl), vl);+ vint16m1_t aux2 = __riscv_vredsum_vs_i16m2_i16m1(va_1, vzero, vl);++ sumf += d*scales[1]*__riscv_vmv_x_s_i16m1_i16(aux2);}@@ -4583,87 +6823,47 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r#else- float sumf = 0;+ uint8_t aux8[QK_K];+ int16_t aux16[16];+ float sums [8];+ memset(sums, 0, 8*sizeof(float));- int isum[QK_K/16];+ uint16_t s16[2];+ const uint8_t * restrict scales = (const uint8_t *)s16;+ float sumf = 0;for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ uint8_t * restrict a = aux8;+ for (int l = 0; l < 32; ++l) a[l+ 0] = q4[l] & 0xF;+ for (int l = 0; l < 32; ++l) a[l+32] = q4[l] >> 4;- const uint8_t * q2 = x[i].qs;- const int8_t * q8 = y[i].qs;- const uint8_t * sc = x[i].scales;+ const uint16_t * restrict b = (const uint16_t *)x[i].scales;+ s16[0] = b[0] & 0x0f0f;+ s16[1] = (b[0] >> 4) & 0x0f0f;- int summs = 0;- for (int j = 0; j < QK_K/16; ++j) {- summs += y[i].bsums[j] * (sc[j] >> 4);- }+ sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);- memset(isum, 0, (QK_K/16)*sizeof(int));- for (int l = 0; l < 16; ++l) {- isum[0] += q8[l+ 0] * ((q2[l] >> 0) & 3);- isum[1] += q8[l+16] * ((q2[l] >> 2) & 3);- isum[2] += q8[l+32] * ((q2[l] >> 4) & 3);- isum[3] += q8[l+48] * ((q2[l] >> 6) & 3);- }- for (int l = 0; l < QK_K/16; ++l) {- isum[l] *= (sc[l] & 0xF);+ for (int j = 0; j < QK_K/32; ++j) {+ for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l];+ q8 += 16; a += 16;+ for (int l = 0; l < 16; ++l) aux16[l] += q8[l] * a[l];+ q8 += 16; a += 16;+ const float dl = d * scales[j];+ for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[l+8]);}- sumf += dall * (isum[0] + isum[1] + isum[2] + isum[3]) - dmin * summs;}+ for (int l = 0; l < 8; ++l) sumf += sums[l];*s = sumf;#endif}#endif#if QK_K == 256--#ifdef __x86_64__-#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_default-#include "ggml_vec_dot_q3_K_q8_K_256.inc"-#undef ggml_vec_dot_q3_K_q8_K-#define ggml_vec_dot_q3_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q3_K_q8_K_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "ggml_vec_dot_q3_K_q8_K_256.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef ggml_vec_dot_q3_K_q8_K-#define ggml_vec_dot_q3_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q3_K_q8_K_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_q3_K_q8_K_256.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_q3_K_q8_K-void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q3_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);- } else if (X86_HAVE(AVX)) {- ggml_vec_dot_q3_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);- } else {- ggml_vec_dot_q3_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);- }-}-#else-#define static-#include "ggml_vec_dot_q3_K_q8_K_256.inc"-#undef static-#endif--#else--void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);assert(nrc == 1);UNUSED(nrc);@@ -4671,390 +6871,424 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * rUNUSED(by);UNUSED(bs);- const block_q3_K * restrict x = vx;+ const block_q5_K * restrict x = vx;const block_q8_K * restrict y = vy;const int nb = n / QK_K;-#ifdef __ARM_NEON- const int32x4_t vzero = vdupq_n_s32(0);+ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;- const uint8x16_t m3b = vdupq_n_u8(0x3);- const uint8x16_t mh = vdupq_n_u8(4);+ uint32_t utmp[4];- ggml_int8x16x4_t q3bytes;+#ifdef __ARM_NEON+ const uint8x16_t m4b = vdupq_n_u8(0xf);+ const uint8x16_t mone = vdupq_n_u8(1);+ const uint8x16_t mtwo = vdupq_n_u8(2);+ const int32x4_t mzero = vdupq_n_s32(0);- uint16_t aux16[2];- int8_t * scales = (int8_t *)aux16;+ ggml_int8x16x4_t q5bytes;- float sum = 0;+ float sumf = 0;for (int i = 0; i < nb; ++i) {- ggml_uint8x16x4_t q3h;+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);- const uint8x8_t hbits = vld1_u8(x[i].hmask);- const uint8x16_t q3bits = vld1q_u8(x[i].qs);- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(y[i].qs);+ const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));- const uint16_t a = *(const uint16_t *)x[i].scales;- aux16[0] = a & 0x0f0f;- aux16[1] = (a >> 4) & 0x0f0f;+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;- for (int j = 0; j < 4; ++j) scales[j] -= 8;+ const uint8x8_t mins8 = vld1_u8((const uint8_t*)utmp + 8);+ const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(mins8));+ const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),+ vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));+ int32_t sumi_mins = vaddvq_s32(prod);- int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);+ const uint8_t * scales = (const uint8_t *)utmp;- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const uint8_t * restrict q5 = x[i].qs;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;- const uint8x16_t htmp = vcombine_u8(hbits, vshr_n_u8(hbits, 1));- q3h.val[0] = vandq_u8(mh, vshlq_n_u8(htmp, 2));- q3h.val[1] = vandq_u8(mh, htmp);- q3h.val[2] = vandq_u8(mh, vshrq_n_u8(htmp, 2));- q3h.val[3] = vandq_u8(mh, vshrq_n_u8(htmp, 4));+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);- q3bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q3bits, m3b), q3h.val[0]));- q3bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 2), m3b), q3h.val[1]));- q3bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 4), m3b), q3h.val[2]));- q3bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q3bits, 6), q3h.val[3]));+ ggml_uint8x16x4_t q5h;- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes.val[0])) * scales[0];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes.val[1])) * scales[2];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes.val[2])) * scales[1];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes.val[3])) * scales[3];+ int32_t sumi = 0;- sum += d * isum;+ for (int j = 0; j < QK_K/64; ++j) {++ const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;++ q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);+ q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);+ q5h.val[2] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[0]), 3);+ q5h.val[3] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[1]), 3);+ qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 2);+ qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 2);++ q5bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[0], m4b), q5h.val[0]));+ q5bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[1], m4b), q5h.val[1]));+ q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));+ q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));++ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;+ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;+ }+ sumf += d * sumi - dmin * sumi_mins;}- *s = sum;+ *s = sumf;#elif defined __AVX2__- const __m256i m3 = _mm256_set1_epi8(3);- const __m256i m1 = _mm256_set1_epi8(1);+ const __m256i m4 = _mm256_set1_epi8(0xF);+ const __m128i mzero = _mm_setzero_si128();+ const __m256i mone = _mm256_set1_epi8(1);__m256 acc = _mm256_setzero_ps();- uint64_t aux64;+ float summs = 0.f;- uint16_t aux16[2];- const int8_t * aux8 = (const int8_t *)aux16;+ for (int i = 0; i < nb; ++i) {- for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q5 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+#if QK_K == 256const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);- const uint8_t * restrict q3 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;+#else+ // TODO+ const float d = 0, dmin = 0;+#endif- const uint16_t a = *(const uint16_t *)x[i].scales;- aux16[0] = a & 0x0f0f;- aux16[1] = (a >> 4) & 0x0f0f;+ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));- const __m256i scale_0 = MM256_SET_M128I(_mm_set1_epi16(aux8[2] - 8), _mm_set1_epi16(aux8[0] - 8));- const __m256i scale_1 = MM256_SET_M128I(_mm_set1_epi16(aux8[3] - 8), _mm_set1_epi16(aux8[1] - 8));+ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);+ summs += dmin * _mm_extract_epi32(hsum, 0);- memcpy(&aux64, x[i].hmask, 8);+ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);+ const __m256i scales = MM256_SET_M128I(sc128, sc128);- const __m128i haux = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);- __m256i q3h_0 = MM256_SET_M128I(_mm_srli_epi16(haux, 2), haux);- __m256i q3h_1 = _mm256_srli_epi16(q3h_0, 4);- q3h_0 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_0, m1), 2);- q3h_1 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_1, m1), 2);+ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);+ __m256i hmask = mone;- // load low 2 bits- const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);+ __m256i sumi = _mm256_setzero_si256();- // prepare low and high bits- const __m256i q3aux = MM256_SET_M128I(_mm_srli_epi16(q3bits, 2), q3bits);- const __m256i q3l_0 = _mm256_and_si256(q3aux, m3);- const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3aux, 4), m3);+ int bit = 0;- // load Q8 quants- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));+ for (int j = 0; j < QK_K/64; ++j) {- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,- // and 2 if the high bit was set)- const __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);- const __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);+ const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));+ const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));- __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);- __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);+ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;- p16_0 = _mm256_sub_epi16(p16_0, q8s_0);- p16_1 = _mm256_sub_epi16(p16_1, q8s_1);+ const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);+ const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);+ const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);+ hmask = _mm256_slli_epi16(hmask, 1);- // multiply with scales- p16_0 = _mm256_madd_epi16(scale_0, p16_0);- p16_1 = _mm256_madd_epi16(scale_1, p16_1);+ const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);+ const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);+ const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);+ hmask = _mm256_slli_epi16(hmask, 1);- p16_0 = _mm256_add_epi32(p16_0, p16_1);+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- // multiply with block scale and accumulate- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16_0), acc);+ __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);++ p16_0 = _mm256_madd_epi16(scale_0, p16_0);+ p16_1 = _mm256_madd_epi16(scale_1, p16_1);++ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));++ }++ __m256 vd = _mm256_set1_ps(d);+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);}- *s = hsum_float_8(acc);+ *s = hsum_float_8(acc) + summs;#elif defined __AVX__- const __m128i m3 = _mm_set1_epi8(3);- const __m128i m1 = _mm_set1_epi8(1);+ const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i mzero = _mm_setzero_si128();+ const __m128i mone = _mm_set1_epi8(1);+ const __m128i m2 = _mm_set1_epi8(2);__m256 acc = _mm256_setzero_ps();- uint64_t aux64;-- uint16_t aux16[2];- const int8_t * aux8 = (const int8_t *)aux16;+ float summs = 0.f;for (int i = 0; i < nb; ++i) {const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);- const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict q5 = x[i].qs;const int8_t * restrict q8 = y[i].qs;- const uint16_t a = *(const uint16_t *)x[i].scales;- aux16[0] = a & 0x0f0f;- aux16[1] = (a >> 4) & 0x0f0f;-- const __m128i scale_0 = _mm_set1_epi16(aux8[0] - 8);- const __m128i scale_1 = _mm_set1_epi16(aux8[2] - 8);- const __m128i scale_2 = _mm_set1_epi16(aux8[1] - 8);- const __m128i scale_3 = _mm_set1_epi16(aux8[3] - 8);-- memcpy(&aux64, x[i].hmask, 8);-- __m128i q3h_0 = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);- __m128i q3h_1 = _mm_srli_epi16(q3h_0, 2);- __m128i q3h_2 = _mm_srli_epi16(q3h_0, 4);- __m128i q3h_3 = _mm_srli_epi16(q3h_0, 6);- q3h_0 = _mm_slli_epi16(_mm_andnot_si128(q3h_0, m1), 2);- q3h_1 = _mm_slli_epi16(_mm_andnot_si128(q3h_1, m1), 2);- q3h_2 = _mm_slli_epi16(_mm_andnot_si128(q3h_2, m1), 2);- q3h_3 = _mm_slli_epi16(_mm_andnot_si128(q3h_3, m1), 2);-- // load low 2 bits- const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);-- // prepare low and high bits- const __m128i q3l_0 = _mm_and_si128(q3bits, m3);- const __m128i q3l_1 = _mm_and_si128(_mm_srli_epi16(q3bits, 2), m3);- const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits, 4), m3);- const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits, 6), m3);+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;- // load Q8 quants- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));+ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);+ const __m128i scales = _mm_cvtepu8_epi16(utmps);+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm_maddubs_epi16,- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,- // and 2 if the high bit was set)- const __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, _mm256_extractf128_si256(q8_0, 0));- const __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, _mm256_extractf128_si256(q8_0, 1));- const __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, _mm256_extractf128_si256(q8_1, 0));- const __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, _mm256_extractf128_si256(q8_1, 1));+ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);+ const __m128i prod = _mm_madd_epi16(mins, q8s);+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);+ summs += dmin * _mm_extract_epi32(hsum, 0);- __m128i p16_0 = _mm_maddubs_epi16(q3l_0, _mm256_extractf128_si256(q8_0, 0));- __m128i p16_1 = _mm_maddubs_epi16(q3l_1, _mm256_extractf128_si256(q8_0, 1));- __m128i p16_2 = _mm_maddubs_epi16(q3l_2, _mm256_extractf128_si256(q8_1, 0));- __m128i p16_3 = _mm_maddubs_epi16(q3l_3, _mm256_extractf128_si256(q8_1, 1));+ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);+ __m128i hmask = mone;- p16_0 = _mm_sub_epi16(p16_0, q8s_0);- p16_1 = _mm_sub_epi16(p16_1, q8s_1);- p16_2 = _mm_sub_epi16(p16_2, q8s_2);- p16_3 = _mm_sub_epi16(p16_3, q8s_3);+ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();- // multiply with scales- p16_0 = _mm_madd_epi16(scale_0, p16_0);- p16_1 = _mm_madd_epi16(scale_1, p16_1);- p16_2 = _mm_madd_epi16(scale_2, p16_2);- p16_3 = _mm_madd_epi16(scale_3, p16_3);+ int bit = 0;- p16_0 = _mm_add_epi32(p16_0, p16_2);- p16_1 = _mm_add_epi32(p16_1, p16_3);- __m256i p16 = MM256_SET_M128I(p16_1, p16_0);+ __m128i shuffle = _mm_set1_epi16(0x0100);+ for (int j = 0; j < QK_K/64; ++j) {- // multiply with block scale and accumulate- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16)), acc);+ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);++ const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;+ const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;++ __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);+ __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);+ __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);+ __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);+ __m128i q5_0 = _mm_add_epi8(q5l_0, q5h_0);+ __m128i q5_1 = _mm_add_epi8(q5l_1, q5h_1);+ hmask = _mm_slli_epi16(hmask, 1);++ __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);+ __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);+ p16_0 = _mm_madd_epi16(scale_0, p16_0);+ p16_1 = _mm_madd_epi16(scale_0, p16_1);++ q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);+ q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);+ q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);+ q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);+ q5_0 = _mm_add_epi8(q5l_0, q5h_0);+ q5_1 = _mm_add_epi8(q5l_1, q5h_1);+ hmask = _mm_slli_epi16(hmask, 1);++ q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);+ __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);+ p16_2 = _mm_madd_epi16(scale_1, p16_2);+ p16_3 = _mm_madd_epi16(scale_1, p16_3);++ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));++ }++ __m256 vd = _mm256_set1_ps(d);+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);}- *s = hsum_float_8(acc);+ *s = hsum_float_8(acc) + summs;#elif defined __riscv_v_intrinsic- uint16_t aux16[2];- int8_t * scales = (int8_t *)aux16;+ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];float sumf = 0;+ float sums = 0.0;++ size_t vl;for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q3 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;+ vl = 8;- const uint16_t a = *(const uint16_t *)x[i].scales;- aux16[0] = a & 0x0f0f;- aux16[1] = (a >> 4) & 0x0f0f;+ const uint8_t * restrict q5 = x[i].qs;+ const uint8_t * restrict hm = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;- for (int j = 0; j < 4; ++j) scales[j] -= 8;+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;- int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);+ vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);+ vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);+ vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);+ vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);+ vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));+ vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);- // load qh- vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(x[i].hmask, 8);- vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));+ vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);+ sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);- size_t vl = 16;+ vl = 32;+ int32_t aux32 = 0;+ int is = 0;- // extend and combine both qh_x1 and qh_x2- vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);+ uint8_t m = 1;+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);+ vuint8m1_t vqh = __riscv_vle8_v_u8m1(hm, vl);- vuint8mf2_t qh_0 = __riscv_vand_vx_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);- vuint8mf2_t qh_1 = __riscv_vand_vx_u8mf2(qh_x, 0x4, vl);- vuint8mf2_t qh_2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);- vuint8mf2_t qh_3 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), 0x4, vl);+ for (int j = 0; j < QK_K/64; ++j) {+ // load Q5 and Q8+ vuint8m1_t q5_x = __riscv_vle8_v_u8m1(q5, vl);+ vint8m1_t q8_y1 = __riscv_vle8_v_i8m1(q8, vl);+ vint8m1_t q8_y2 = __riscv_vle8_v_i8m1(q8+32, vl);- // load Q3- vuint8mf2_t q3_x = __riscv_vle8_v_u8mf2(q3, vl);+ // compute mask for addition+ vint8m1_t q5_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q5_x, 0x0F, vl));+ vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);+ vbool8_t vmask_1 = __riscv_vmsne_vx_u8m1_b8(qh_m1, 0, vl);+ vint8m1_t q5_m1 = __riscv_vadd_vx_i8m1_m(vmask_1, q5_a, 16, vl);+ m <<= 1;- vuint8mf2_t q3h_0 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(q3_x, 0x3, vl), qh_0, vl);- vuint8mf2_t q3h_1 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 2, vl), 0x3, vl), qh_1, vl);- vuint8mf2_t q3h_2 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 4, vl), 0x3, vl), qh_2, vl);- vuint8mf2_t q3h_3 = __riscv_vor_vv_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x6, vl), qh_3, vl);+ vint8m1_t q5_l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q5_x, 0x04, vl));+ vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);+ vbool8_t vmask_2 = __riscv_vmsne_vx_u8m1_b8(qh_m2, 0, vl);+ vint8m1_t q5_m2 = __riscv_vadd_vx_i8m1_m(vmask_2, q5_l, 16, vl);+ m <<= 1;- vint8mf2_t q3_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_0);- vint8mf2_t q3_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_1);- vint8mf2_t q3_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_2);- vint8mf2_t q3_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_3);+ vint16m2_t v0 = __riscv_vwmul_vv_i16m2(q5_m1, q8_y1, vl);+ vint16m2_t v1 = __riscv_vwmul_vv_i16m2(q5_m2, q8_y2, vl);- // load Q8 and take product with Q3- vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q3_0, __riscv_vle8_v_i8mf2(q8, vl), vl);- vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q3_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);- vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q3_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);- vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q3_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);+ vint32m4_t vs1 = __riscv_vwmul_vx_i32m4(v0, scales[is++], vl);+ vint32m4_t vs2 = __riscv_vwmul_vx_i32m4(v1, scales[is++], vl);- vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);- vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);- vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);- vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);+ vint32m1_t vacc1 = __riscv_vredsum_vs_i32m4_i32m1(vs1, vzero, vl);+ vint32m1_t vacc2 = __riscv_vredsum_vs_i32m4_i32m1(vs2, vzero, vl);- isum += __riscv_vmv_x_s_i32m1_i32(vs_0) * scales[0];- isum += __riscv_vmv_x_s_i32m1_i32(vs_1) * scales[2];- isum += __riscv_vmv_x_s_i32m1_i32(vs_2) * scales[1];- isum += __riscv_vmv_x_s_i32m1_i32(vs_3) * scales[3];+ aux32 += __riscv_vmv_x_s_i32m1_i32(vacc1) + __riscv_vmv_x_s_i32m1_i32(vacc2);+ q5 += 32; q8 += 64;- sumf += d * isum;+ }++ vfloat32m1_t vaux = __riscv_vfmul_vf_f32m1(__riscv_vfmv_v_f_f32m1(aux32, 1), d, 1);+ sums += __riscv_vfmv_f_s_f32m1_f32(vaux);}- *s = sumf;+ *s = sumf+sums;#else+ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];+int8_t aux8[QK_K];int16_t aux16[8];float sums [8];int32_t aux32[8];- int32_t scales[4];memset(sums, 0, 8*sizeof(float));float sumf = 0;for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q3 = x[i].qs;- const uint8_t * restrict hm = x[i].hmask;+ const uint8_t * restrict q4 = x[i].qs;+ const uint8_t * restrict hm = x[i].qh;const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));int8_t * restrict a = aux8;- for (int l = 0; l < 8; ++l) {- a[l+ 0] = (int8_t)((q3[l+0] >> 0) & 3) - (hm[l] & 0x01 ? 0 : 4);- a[l+ 8] = (int8_t)((q3[l+8] >> 0) & 3) - (hm[l] & 0x02 ? 0 : 4);- a[l+16] = (int8_t)((q3[l+0] >> 2) & 3) - (hm[l] & 0x04 ? 0 : 4);- a[l+24] = (int8_t)((q3[l+8] >> 2) & 3) - (hm[l] & 0x08 ? 0 : 4);- a[l+32] = (int8_t)((q3[l+0] >> 4) & 3) - (hm[l] & 0x10 ? 0 : 4);- a[l+40] = (int8_t)((q3[l+8] >> 4) & 3) - (hm[l] & 0x20 ? 0 : 4);- a[l+48] = (int8_t)((q3[l+0] >> 6) & 3) - (hm[l] & 0x40 ? 0 : 4);- a[l+56] = (int8_t)((q3[l+8] >> 6) & 3) - (hm[l] & 0x80 ? 0 : 4);- }-- scales[0] = (x[i].scales[0] & 0xF) - 8;- scales[1] = (x[i].scales[0] >> 4) - 8;- scales[2] = (x[i].scales[1] & 0xF) - 8;- scales[3] = (x[i].scales[1] >> 4) - 8;+ uint8_t m = 1;+ for (int j = 0; j < QK_K/64; ++j) {+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);+ a += 32; m <<= 1;+ q4 += 32;+ }+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;- memset(aux32, 0, 8*sizeof(int32_t));- for (int j = 0; j < QK_K/16; ++j) {+ int sumi = 0;+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];+ a = aux8;+ int is = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ int32_t scale = scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] += q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux32[l] += scales[j] * aux16[l];}const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;+ sumf -= dmin * sumi;}for (int l = 0; l < 8; ++l) sumf += sums[l];*s = sumf;-#endif-}-#endif--#if QK_K == 256-#ifdef __x86_64__-#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_default-#include "ggml_vec_dot_q4_K_q8_K_256.inc"-#undef ggml_vec_dot_q4_K_q8_K-#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q4_K_q8_K_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "ggml_vec_dot_q4_K_q8_K_256.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef ggml_vec_dot_q4_K_q8_K-#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_K_q8_K_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_q4_K_q8_K_256.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_q4_K_q8_K-void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q4_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);- } else if (X86_HAVE(AVX)) {- ggml_vec_dot_q4_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);- } else {- ggml_vec_dot_q4_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);- }-}#else-#define static-#include "ggml_vec_dot_q4_K_q8_K_256.inc"-#undef static-#endif-#else-void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);assert(nrc == 1);UNUSED(nrc);@@ -5062,199 +7296,224 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * rUNUSED(by);UNUSED(bs);- const block_q4_K * restrict x = vx;+ const block_q5_K * restrict x = vx;const block_q8_K * restrict y = vy;const int nb = n / QK_K;#ifdef __ARM_NEONconst uint8x16_t m4b = vdupq_n_u8(0xf);-+ const uint8x16_t mh = vdupq_n_u8(16);const int32x4_t mzero = vdupq_n_s32(0);- float sumf = 0;-- ggml_int8x16x2_t q4bytes;- ggml_int8x16x4_t q8bytes;-- float sum_mins = 0.f;+ ggml_int8x16x4_t q5bytes;+ ggml_uint8x16x4_t q5h;- uint16_t aux16[2];- const uint8_t * restrict scales = (const uint8_t *)aux16;+ float sumf = 0;for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- const uint16_t * restrict a = (const uint16_t *)x[i].scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;-- const int32_t summi = scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]);- sum_mins += y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * summi;+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const int8_t * sc = x[i].scales;- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);+ const uint8_t * restrict q5 = x[i].qs;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;- const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4);+ const uint8x8_t qhbits = vld1_u8(qh);- q8bytes = ggml_vld1q_s8_x4(q8);- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));+ const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5);+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);- const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);- const int32_t sumi1 = vaddvq_s32(p1) * scales[0];+ const uint8x16_t htmp = vcombine_u8(qhbits, vshr_n_u8(qhbits, 1));+ q5h.val[0] = vbicq_u8(mh, vshlq_n_u8(htmp, 4));+ q5h.val[1] = vbicq_u8(mh, vshlq_n_u8(htmp, 2));+ q5h.val[2] = vbicq_u8(mh, htmp);+ q5h.val[3] = vbicq_u8(mh, vshrq_n_u8(htmp, 2));- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));+ q5bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[0], m4b)), vreinterpretq_s8_u8(q5h.val[0]));+ q5bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[1], m4b)), vreinterpretq_s8_u8(q5h.val[1]));+ q5bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[0], 4)), vreinterpretq_s8_u8(q5h.val[2]));+ q5bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[1], 4)), vreinterpretq_s8_u8(q5h.val[3]));- const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[2]), q4bytes.val[1], q8bytes.val[3]);- const int32_t sumi2 = vaddvq_s32(p2) * scales[1];+ int32_t sumi1 = sc[0] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]));+ int32_t sumi2 = sc[1] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[1], q8bytes.val[1]));+ int32_t sumi3 = sc[2] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]));+ int32_t sumi4 = sc[3] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[3], q8bytes.val[3]));- sumf += d * (sumi1 + sumi2);+ sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);}- *s = sumf - sum_mins;+ *s = sumf;#elif defined __AVX2__const __m256i m4 = _mm256_set1_epi8(0xF);+ const __m256i mone = _mm256_set1_epi8(1);__m256 acc = _mm256_setzero_ps();- float summs = 0;+ for (int i = 0; i < nb; ++i) {- uint16_t aux16[2];- const uint8_t * scales = (const uint8_t *)aux16;+ const uint8_t * restrict q5 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;- for (int i = 0; i < nb; ++i) {+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;- const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;- const __m256 vd = _mm256_set1_ps(d);+ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);- const uint16_t * a = (const uint16_t *)x[i].scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;+ const __m256i scale_l = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[1]), _mm_set1_epi16(x[i].scales[0]));+ const __m256i scale_h = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[3]), _mm_set1_epi16(x[i].scales[2]));- summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));+ int64_t aux64;+ memcpy(&aux64, x[i].qh, 8);+ const __m128i haux128 = _mm_set_epi64x(aux64 >> 1, aux64);+ const __m256i haux256 = MM256_SET_M128I(_mm_srli_epi16(haux128, 2), haux128);- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;+ const __m256i q5h_0 = _mm256_slli_epi16(_mm256_andnot_si256(haux256, mone), 4);+ const __m256i q5h_1 = _mm256_slli_epi16(_mm256_andnot_si256(_mm256_srli_epi16(haux256, 4), mone), 4);- const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);- const __m256i q4l = _mm256_and_si256(q4bits, m4);- const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);+ const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);+ const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);- const __m256i q8l = _mm256_loadu_si256((const __m256i*)(q8+ 0));- const __m256i q8h = _mm256_loadu_si256((const __m256i*)(q8+32));+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));- const __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);- const __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);+ const __m256i p16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5l_0, q8_0));+ const __m256i p16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5l_1, q8_1));+ const __m256i s16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5h_0, q8_0));+ const __m256i s16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5h_1, q8_1));- const __m256i p32l = _mm256_madd_epi16(_mm256_set1_epi16(scales[0]), p16l);- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32l), acc);+ const __m256i dot = _mm256_sub_epi32(_mm256_add_epi32(p16_0, p16_1), _mm256_add_epi32(s16_0, s16_1));- const __m256i p32h = _mm256_madd_epi16(_mm256_set1_epi16(scales[1]), p16h);- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32h), acc);+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(dot), acc);}- *s = hsum_float_8(acc) - summs;+ *s = hsum_float_8(acc);#elif defined __AVX__const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i mone = _mm_set1_epi8(1);__m256 acc = _mm256_setzero_ps();- float summs = 0;+ for (int i = 0; i < nb; ++i) {- uint16_t aux16[2];- const uint8_t * scales = (const uint8_t *)aux16;+ const uint8_t * restrict q5 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;- for (int i = 0; i < nb; ++i) {+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;- const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;- const __m256 vd = _mm256_set1_ps(d);+ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);- const uint16_t * a = (const uint16_t *)x[i].scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;+ const __m128i scale_0 = _mm_set1_epi16(x[i].scales[0]);+ const __m128i scale_1 = _mm_set1_epi16(x[i].scales[1]);+ const __m128i scale_2 = _mm_set1_epi16(x[i].scales[2]);+ const __m128i scale_3 = _mm_set1_epi16(x[i].scales[3]);- summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));+ int64_t aux64;+ memcpy(&aux64, x[i].qh, 8);+ const __m128i haux128_0 = _mm_set_epi64x(aux64 >> 1, aux64);+ const __m128i haux128_1 = _mm_srli_epi16(haux128_0, 2);- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;+ const __m128i q5h_0 = _mm_slli_epi16(_mm_andnot_si128(haux128_0, mone), 4);+ const __m128i q5h_1 = _mm_slli_epi16(_mm_andnot_si128(haux128_1, mone), 4);+ const __m128i q5h_2 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_0, 4), mone), 4);+ const __m128i q5h_3 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_1, 4), mone), 4);- const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);- const __m128i q4bits_0 = _mm256_extractf128_si256(q4bits, 0);- const __m128i q4bits_1 = _mm256_extractf128_si256(q4bits, 1);- const __m128i q4_0 = _mm_and_si128(q4bits_0, m4);- const __m128i q4_1 = _mm_and_si128(q4bits_1, m4);- const __m128i q4_2 = _mm_and_si128(_mm_srli_epi16(q4bits_0, 4), m4);- const __m128i q4_3 = _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4);+ const __m128i q5l_0 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 0), m4);+ const __m128i q5l_1 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 1), m4);+ const __m128i q5l_2 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 0), 4), m4);+ const __m128i q5l_3 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 1), 4), m4);const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));- const __m128i p16_0 = _mm_maddubs_epi16(q4_0, _mm256_extractf128_si256(q8_0, 0));- const __m128i p16_1 = _mm_maddubs_epi16(q4_1, _mm256_extractf128_si256(q8_0, 1));- const __m128i p16_2 = _mm_maddubs_epi16(q4_2, _mm256_extractf128_si256(q8_1, 0));- const __m128i p16_3 = _mm_maddubs_epi16(q4_3, _mm256_extractf128_si256(q8_1, 1));+ const __m128i p16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5l_0, _mm256_extractf128_si256(q8_0, 0)));+ const __m128i p16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5l_1, _mm256_extractf128_si256(q8_0, 1)));+ const __m128i p16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5l_2, _mm256_extractf128_si256(q8_1, 0)));+ const __m128i p16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5l_3, _mm256_extractf128_si256(q8_1, 1)));+ const __m128i s16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5h_0, _mm256_extractf128_si256(q8_0, 0)));+ const __m128i s16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5h_1, _mm256_extractf128_si256(q8_0, 1)));+ const __m128i s16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5h_2, _mm256_extractf128_si256(q8_1, 0)));+ const __m128i s16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5h_3, _mm256_extractf128_si256(q8_1, 1)));- const __m128i p32_0 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_0);- const __m128i p32_1 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_1);- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_1, p32_0))), acc);+ const __m128i dot_0 = _mm_sub_epi32(_mm_add_epi32(p16_0, p16_2), _mm_add_epi32(s16_0, s16_2));+ const __m128i dot_1 = _mm_sub_epi32(_mm_add_epi32(p16_1, p16_3), _mm_add_epi32(s16_1, s16_3));- const __m128i p32_2 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_2);- const __m128i p32_3 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_3);- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_3, p32_2))), acc);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(dot_1, dot_0))), acc);}- *s = hsum_float_8(acc) - summs;+ *s = hsum_float_8(acc);#elif defined __riscv_v_intrinsic- uint16_t s16[2];- const uint8_t * restrict scales = (const uint8_t *)s16;-float sumf = 0;for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const int8_t * sc = x[i].scales;- const uint16_t * restrict b = (const uint16_t *)x[i].scales;- s16[0] = b[0] & 0x0f0f;- s16[1] = (b[0] >> 4) & 0x0f0f;+ const uint8_t * restrict q5 = x[i].qs;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;- sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);- size_t vl = 32;+ // load qh+ vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(qh, 8);+ vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));- vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);+ size_t vl = 16;- // load Q4- vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);+ // combine both qh_1 and qh_2+ vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);- // load Q8 and multiply it with lower Q4 nibble- vint8m1_t q4_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));- vint16m2_t va_0 = __riscv_vwmul_vv_i16m2(q4_a, __riscv_vle8_v_i8m1(q8, vl), vl);- vint16m1_t aux1 = __riscv_vredsum_vs_i16m2_i16m1(va_0, vzero, vl);+ vuint8mf2_t qh_h0 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);+ vuint8mf2_t qh_h1 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), vl), 16, vl);+ vuint8mf2_t qh_h2 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(qh_x, vl), 16, vl);+ vuint8mf2_t qh_h3 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);- sumf += d*scales[0]*__riscv_vmv_x_s_i16m1_i16(aux1);+ vint8mf2_t qh_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h0);+ vint8mf2_t qh_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h1);+ vint8mf2_t qh_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h2);+ vint8mf2_t qh_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h3);- // load Q8 and multiply it with upper Q4 nibble- vint8m1_t q4_s = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));- vint16m2_t va_1 = __riscv_vwmul_vv_i16m2(q4_s, __riscv_vle8_v_i8m1(q8+32, vl), vl);- vint16m1_t aux2 = __riscv_vredsum_vs_i16m2_i16m1(va_1, vzero, vl);+ // load q5+ vuint8mf2_t q5_x1 = __riscv_vle8_v_u8mf2(q5, vl);+ vuint8mf2_t q5_x2 = __riscv_vle8_v_u8mf2(q5+16, vl);- sumf += d*scales[1]*__riscv_vmv_x_s_i16m1_i16(aux2);+ vint8mf2_t q5s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x1, 0xF, vl));+ vint8mf2_t q5s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x2, 0xF, vl));+ vint8mf2_t q5s_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x1, 0x4, vl));+ vint8mf2_t q5s_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x2, 0x4, vl));++ vint8mf2_t q5_0 = __riscv_vsub_vv_i8mf2(q5s_0, qh_0, vl);+ vint8mf2_t q5_1 = __riscv_vsub_vv_i8mf2(q5s_1, qh_1, vl);+ vint8mf2_t q5_2 = __riscv_vsub_vv_i8mf2(q5s_2, qh_2, vl);+ vint8mf2_t q5_3 = __riscv_vsub_vv_i8mf2(q5s_3, qh_3, vl);++ // load Q8 and multiply it with Q5+ vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q5_0, __riscv_vle8_v_i8mf2(q8, vl), vl);+ vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q5_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);+ vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q5_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);+ vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q5_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);++ vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);+ vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);+ vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);+ vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);++ int32_t sumi1 = sc[0] * __riscv_vmv_x_s_i32m1_i32(vs_0);+ int32_t sumi2 = sc[1] * __riscv_vmv_x_s_i32m1_i32(vs_1);+ int32_t sumi3 = sc[2] * __riscv_vmv_x_s_i32m1_i32(vs_2);+ int32_t sumi4 = sc[3] * __riscv_vmv_x_s_i32m1_i32(vs_3);++ sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);}@@ -5262,37 +7521,34 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r#else- uint8_t aux8[QK_K];+ int8_t aux8[QK_K];int16_t aux16[16];float sums [8];memset(sums, 0, 8*sizeof(float));- uint16_t s16[2];- const uint8_t * restrict scales = (const uint8_t *)s16;-float sumf = 0;for (int i = 0; i < nb; ++i) {const uint8_t * restrict q4 = x[i].qs;+ const uint8_t * restrict hm = x[i].qh;const int8_t * restrict q8 = y[i].qs;- uint8_t * restrict a = aux8;- for (int l = 0; l < 32; ++l) a[l+ 0] = q4[l] & 0xF;- for (int l = 0; l < 32; ++l) a[l+32] = q4[l] >> 4;-- const uint16_t * restrict b = (const uint16_t *)x[i].scales;- s16[0] = b[0] & 0x0f0f;- s16[1] = (b[0] >> 4) & 0x0f0f;-- sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));+ int8_t * restrict a = aux8;+ for (int l = 0; l < 32; ++l) {+ a[l+ 0] = q4[l] & 0xF;+ a[l+32] = q4[l] >> 4;+ }+ for (int is = 0; is < 8; ++is) {+ uint8_t m = 1 << is;+ for (int l = 0; l < 8; ++l) a[8*is + l] -= (hm[l] & m ? 0 : 16);+ }- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const int8_t * restrict sc = x[i].scales;- for (int j = 0; j < QK_K/32; ++j) {+ for (int j = 0; j < QK_K/16; ++j) {+ const float dl = d * sc[j];for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[8+l]);q8 += 16; a += 16;- for (int l = 0; l < 16; ++l) aux16[l] += q8[l] * a[l];- q8 += 16; a += 16;- const float dl = d * scales[j];- for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[l+8]);}}for (int l = 0; l < 8; ++l) sumf += sums[l];@@ -5301,52 +7557,9 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r}#endif-#if QK_K == 256--#ifdef __x86_64__-#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_default-#include "ggml_vec_dot_q5_K_q8_K_256.inc"-#undef ggml_vec_dot_q5_K_q8_K-#define ggml_vec_dot_q5_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q5_K_q8_K_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "ggml_vec_dot_q5_K_q8_K_256.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef ggml_vec_dot_q5_K_q8_K-#define ggml_vec_dot_q5_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q5_K_q8_K_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_q5_K_q8_K_256.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_q5_K_q8_K-void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q5_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);- } else if (X86_HAVE(AVX)) {- ggml_vec_dot_q5_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);- } else {- ggml_vec_dot_q5_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);- }-}-#else-#define static-#include "ggml_vec_dot_q5_K_q8_K_256.inc"-#undef static-#endif--#else-void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+#if QK_K == 256+void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);assert(nrc == 1);UNUSED(nrc);@@ -5354,98 +7567,181 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * rUNUSED(by);UNUSED(bs);- const block_q5_K * restrict x = vx;+ const block_q6_K * restrict x = vx;const block_q8_K * restrict y = vy;const int nb = n / QK_K;#ifdef __ARM_NEON- const uint8x16_t m4b = vdupq_n_u8(0xf);- const uint8x16_t mh = vdupq_n_u8(16);- const int32x4_t mzero = vdupq_n_s32(0);+ float sum = 0;- ggml_int8x16x4_t q5bytes;- ggml_uint8x16x4_t q5h;+ const uint8x16_t m4b = vdupq_n_u8(0xF);+ const int32x4_t vzero = vdupq_n_s32(0);+ //const int8x16_t m32s = vdupq_n_s8(32);- float sumf = 0;+ const uint8x16_t mone = vdupq_n_u8(3);++ ggml_int8x16x4_t q6bytes;+ ggml_uint8x16x4_t q6h;for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const int8_t * sc = x[i].scales;+ const float d_all = GGML_FP16_TO_FP32(x[i].d);- const uint8_t * restrict q5 = x[i].qs;+ const uint8_t * restrict q6 = x[i].ql;const uint8_t * restrict qh = x[i].qh;const int8_t * restrict q8 = y[i].qs;- const uint8x8_t qhbits = vld1_u8(qh);+ const int8_t * restrict scale = x[i].scales;- const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5);- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);+ const int8x16_t scales = vld1q_s8(scale);+ const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};- const uint8x16_t htmp = vcombine_u8(qhbits, vshr_n_u8(qhbits, 1));- q5h.val[0] = vbicq_u8(mh, vshlq_n_u8(htmp, 4));- q5h.val[1] = vbicq_u8(mh, vshlq_n_u8(htmp, 2));- q5h.val[2] = vbicq_u8(mh, htmp);- q5h.val[3] = vbicq_u8(mh, vshrq_n_u8(htmp, 2));+ const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),+ vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),+ vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),+ vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));+ int32_t isum_mins = vaddvq_s32(prod);- q5bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[0], m4b)), vreinterpretq_s8_u8(q5h.val[0]));- q5bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[1], m4b)), vreinterpretq_s8_u8(q5h.val[1]));- q5bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[0], 4)), vreinterpretq_s8_u8(q5h.val[2]));- q5bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[1], 4)), vreinterpretq_s8_u8(q5h.val[3]));+ int32_t isum = 0;- int32_t sumi1 = sc[0] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]));- int32_t sumi2 = sc[1] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[1], q8bytes.val[1]));- int32_t sumi3 = sc[2] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]));- int32_t sumi4 = sc[3] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[3], q8bytes.val[3]));+ for (int j = 0; j < QK_K/128; ++j) {- sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);- }+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;+ ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;+ ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;++ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);+ uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[1], 2);+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);++ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));++ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];++ scale += 4;++ q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;++ shifted = vshrq_n_u8(qhbits.val[0], 4);+ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[1], 4);+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[0], 6);+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[1], 6);+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);++ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));++ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];+ scale += 4;+ }+ //sum += isum * d_all * y[i].d;+ sum += d_all * y[i].d * (isum - 32 * isum_mins);- *s = sumf;+ }+ *s = sum;#elif defined __AVX2__const __m256i m4 = _mm256_set1_epi8(0xF);- const __m256i mone = _mm256_set1_epi8(1);+ const __m256i m2 = _mm256_set1_epi8(3);+ const __m256i m32s = _mm256_set1_epi8(32);__m256 acc = _mm256_setzero_ps();for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q5 = x[i].qs;+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;const int8_t * restrict q8 = y[i].qs;- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);- const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);+ __m256i sumi = _mm256_setzero_si256();- const __m256i scale_l = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[1]), _mm_set1_epi16(x[i].scales[0]));- const __m256i scale_h = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[3]), _mm_set1_epi16(x[i].scales[2]));+ int is = 0;- int64_t aux64;- memcpy(&aux64, x[i].qh, 8);- const __m128i haux128 = _mm_set_epi64x(aux64 >> 1, aux64);- const __m256i haux256 = MM256_SET_M128I(_mm_srli_epi16(haux128, 2), haux128);+ for (int j = 0; j < QK_K/128; ++j) {- const __m256i q5h_0 = _mm256_slli_epi16(_mm256_andnot_si256(haux256, mone), 4);- const __m256i q5h_1 = _mm256_slli_epi16(_mm256_andnot_si256(_mm256_srli_epi16(haux256, 4), mone), 4);+ const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));+ const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));+ const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));+ const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));+ is += 4;- const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);- const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);+ const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));+ const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);+ const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);+ const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);+ const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);- const __m256i p16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5l_0, q8_0));- const __m256i p16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5l_1, q8_1));- const __m256i s16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5h_0, q8_0));- const __m256i s16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5h_1, q8_1));+ const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);+ const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);+ const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);+ const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);+ __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);+ __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);+ __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);++ __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);+ __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);+ __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);++ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);- const __m256i dot = _mm256_sub_epi32(_mm256_add_epi32(p16_0, p16_1), _mm256_add_epi32(s16_0, s16_1));+ p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);+ p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);+ p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);+ p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);- acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(dot), acc);+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));++ }+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);}*s = hsum_float_8(acc);@@ -5453,56 +7749,117 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r#elif defined __AVX__const __m128i m4 = _mm_set1_epi8(0xF);- const __m128i mone = _mm_set1_epi8(1);+ const __m128i m3 = _mm_set1_epi8(3);+ const __m128i m32s = _mm_set1_epi8(32);+ const __m128i m2 = _mm_set1_epi8(2);__m256 acc = _mm256_setzero_ps();for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q5 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);-- const __m128i scale_0 = _mm_set1_epi16(x[i].scales[0]);- const __m128i scale_1 = _mm_set1_epi16(x[i].scales[1]);- const __m128i scale_2 = _mm_set1_epi16(x[i].scales[2]);- const __m128i scale_3 = _mm_set1_epi16(x[i].scales[3]);+ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;- int64_t aux64;- memcpy(&aux64, x[i].qh, 8);- const __m128i haux128_0 = _mm_set_epi64x(aux64 >> 1, aux64);- const __m128i haux128_1 = _mm_srli_epi16(haux128_0, 2);+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);- const __m128i q5h_0 = _mm_slli_epi16(_mm_andnot_si128(haux128_0, mone), 4);- const __m128i q5h_1 = _mm_slli_epi16(_mm_andnot_si128(haux128_1, mone), 4);- const __m128i q5h_2 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_0, 4), mone), 4);- const __m128i q5h_3 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_1, 4), mone), 4);+ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();- const __m128i q5l_0 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 0), m4);- const __m128i q5l_1 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 1), m4);- const __m128i q5l_2 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 0), 4), m4);- const __m128i q5l_3 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 1), 4), m4);+ __m128i shuffle = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);+ for (int j = 0; j < QK_K/128; ++j) {- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));+ const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;+ const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;++ const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);+ const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);+ const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 2), m3), 4);+ const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 2), m3), 4);+ const __m128i q4h_4 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 4), m3), 4);+ const __m128i q4h_5 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 4), m3), 4);+ const __m128i q4h_6 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 6), m3), 4);+ const __m128i q4h_7 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 6), m3), 4);++ const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;++ const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m4), q4h_0);+ const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m4), q4h_1);+ const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m4), q4h_2);+ const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m4), q4h_3);+ const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m4), q4h_4);+ const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m4), q4h_5);+ const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m4), q4h_6);+ const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m4), q4h_7);- const __m128i p16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5l_0, _mm256_extractf128_si256(q8_0, 0)));- const __m128i p16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5l_1, _mm256_extractf128_si256(q8_0, 1)));- const __m128i p16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5l_2, _mm256_extractf128_si256(q8_1, 0)));- const __m128i p16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5l_3, _mm256_extractf128_si256(q8_1, 1)));- const __m128i s16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5h_0, _mm256_extractf128_si256(q8_0, 0)));- const __m128i s16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5h_1, _mm256_extractf128_si256(q8_0, 1)));- const __m128i s16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5h_2, _mm256_extractf128_si256(q8_1, 0)));- const __m128i s16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5h_3, _mm256_extractf128_si256(q8_1, 1)));+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i dot_0 = _mm_sub_epi32(_mm_add_epi32(p16_0, p16_2), _mm_add_epi32(s16_0, s16_2));- const __m128i dot_1 = _mm_sub_epi32(_mm_add_epi32(p16_1, p16_3), _mm_add_epi32(s16_1, s16_3));+ __m128i q8s_0 = _mm_maddubs_epi16(m32s, q8_0);+ __m128i q8s_1 = _mm_maddubs_epi16(m32s, q8_1);+ __m128i q8s_2 = _mm_maddubs_epi16(m32s, q8_2);+ __m128i q8s_3 = _mm_maddubs_epi16(m32s, q8_3);+ __m128i q8s_4 = _mm_maddubs_epi16(m32s, q8_4);+ __m128i q8s_5 = _mm_maddubs_epi16(m32s, q8_5);+ __m128i q8s_6 = _mm_maddubs_epi16(m32s, q8_6);+ __m128i q8s_7 = _mm_maddubs_epi16(m32s, q8_7);++ __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);+ __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);+ __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);+ __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);+ __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);+ __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);+ __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);+ __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);++ p16_0 = _mm_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);++ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);+ const __m128i scale_2 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);+ const __m128i scale_3 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);++ p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);+ p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1);+ p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);+ p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3);+ p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);+ p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_2, scale_2)), p16_5);+ p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);+ p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_3, scale_3)), p16_7);++ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(dot_1, dot_0))), acc);+ }+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);}*s = hsum_float_8(acc);@@ -5510,68 +7867,83 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r#elif defined __riscv_v_intrinsicfloat sumf = 0;-for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const int8_t * sc = x[i].scales;+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- const uint8_t * restrict q5 = x[i].qs;+ const uint8_t * restrict q6 = x[i].ql;const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const int8_t * restrict scale = x[i].scales;++ size_t vl;vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);- // load qh- vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(qh, 8);- vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));+ int sum_t = 0;+ int is = 0;- size_t vl = 16;+ for (int j = 0; j < QK_K/128; ++j) {- // combine both qh_1 and qh_2- vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);+ vl = 32;- vuint8mf2_t qh_h0 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);- vuint8mf2_t qh_h1 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), vl), 16, vl);- vuint8mf2_t qh_h2 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(qh_x, vl), 16, vl);- vuint8mf2_t qh_h3 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);+ // load qh+ vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);- vint8mf2_t qh_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h0);- vint8mf2_t qh_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h1);- vint8mf2_t qh_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h2);- vint8mf2_t qh_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h3);+ // load Q6+ vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);+ vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);- // load q5- vuint8mf2_t q5_x1 = __riscv_vle8_v_u8mf2(q5, vl);- vuint8mf2_t q5_x2 = __riscv_vle8_v_u8mf2(q5+16, vl);+ vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);+ vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);+ vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);+ vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);- vint8mf2_t q5s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x1, 0xF, vl));- vint8mf2_t q5s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x2, 0xF, vl));- vint8mf2_t q5s_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x1, 0x4, vl));- vint8mf2_t q5s_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x2, 0x4, vl));+ vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);+ vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);+ vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);+ vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);- vint8mf2_t q5_0 = __riscv_vsub_vv_i8mf2(q5s_0, qh_0, vl);- vint8mf2_t q5_1 = __riscv_vsub_vv_i8mf2(q5s_1, qh_1, vl);- vint8mf2_t q5_2 = __riscv_vsub_vv_i8mf2(q5s_2, qh_2, vl);- vint8mf2_t q5_3 = __riscv_vsub_vv_i8mf2(q5s_3, qh_3, vl);+ vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);+ vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);+ vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);+ vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);- // load Q8 and multiply it with Q5- vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q5_0, __riscv_vle8_v_i8mf2(q8, vl), vl);- vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q5_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);- vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q5_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);- vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q5_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);+ vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);+ vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);+ vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);+ vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);- vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);- vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);- vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);- vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);+ // load Q8 and take product+ vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);+ vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);+ vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);+ vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);- int32_t sumi1 = sc[0] * __riscv_vmv_x_s_i32m1_i32(vs_0);- int32_t sumi2 = sc[1] * __riscv_vmv_x_s_i32m1_i32(vs_1);- int32_t sumi3 = sc[2] * __riscv_vmv_x_s_i32m1_i32(vs_2);- int32_t sumi4 = sc[3] * __riscv_vmv_x_s_i32m1_i32(vs_3);+ vl = 16;- sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);+ vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);+ vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);+ vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);+ vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);+ vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);+ vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);+ vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);+ vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);++ vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);+ vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);+ vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);++ sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);++ q6 += 64; qh += 32; q8 += 128; is=8;++ }++ sumf += d * sum_t;}@@ -5579,85 +7951,48 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r#else- int8_t aux8[QK_K];- int16_t aux16[16];+ int8_t aux8[QK_K];+ int16_t aux16[8];float sums [8];+ int32_t aux32[8];memset(sums, 0, 8*sizeof(float));float sumf = 0;for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].qs;- const uint8_t * restrict hm = x[i].qh;+ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));int8_t * restrict a = aux8;- for (int l = 0; l < 32; ++l) {- a[l+ 0] = q4[l] & 0xF;- a[l+32] = q4[l] >> 4;- }- for (int is = 0; is < 8; ++is) {- uint8_t m = 1 << is;- for (int l = 0; l < 8; ++l) a[8*is + l] -= (hm[l] & m ? 0 : 16);+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;+ a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;+ a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;+ a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;+ }+ a += 128;+ q4 += 64;+ qh += 32;}-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const int8_t * restrict sc = x[i].scales;-+ a = aux8;+ int is = 0;for (int j = 0; j < QK_K/16; ++j) {- const float dl = d * sc[j];- for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[8+l]);- q8 += 16; a += 16;+ int scale = x[i].scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;}+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];}for (int l = 0; l < 8; ++l) sumf += sums[l];*s = sumf;#endif}-#endif---#if QK_K == 256--#ifdef __x86_64__-#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_default-#include "ggml_vec_dot_q6_K_q8_K_256.inc"-#undef ggml_vec_dot_q6_K_q8_K-#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q6_K_q8_K_avx-#ifndef __AVX__-#define __UNDEF_AVX__-#define __AVX__-#endif-#include "ggml_vec_dot_q6_K_q8_K_256.inc"-#ifdef __UNDEF_AVX__-#undef __UNDEF_AVX__-#undef __AVX__-#endif-#undef ggml_vec_dot_q6_K_q8_K-#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q6_K_q8_K_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_q6_K_q8_K_256.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_q6_K_q8_K-void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q6_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);- } else if (X86_HAVE(AVX)) {- ggml_vec_dot_q6_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);- } else {- ggml_vec_dot_q6_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);- }-}-#else-#define static-#include "ggml_vec_dot_q6_K_q8_K_256.inc"-#undef static-#endif#else@@ -10050,3 +12385,305 @@ void quantize_row_iq2_s(const float * restrict x, void * restrict vy, int64_t k)block_iq2_s * restrict y = vy;quantize_row_iq2_s_reference(x, y, k);}++static bool validate_float(float f, size_t i) {+ if (isinf(f)) {+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);+ return false;+ }++ if (isnan(f)) {+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);+ return false;+ }++ return true;+}++static bool isinf_fp16(ggml_fp16_t f) {+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) == 0;+}++static bool isnan_fp16(ggml_fp16_t f) {+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) != 0;+}++static bool validate_fp16(ggml_fp16_t f, size_t i) {+ if (isinf_fp16(f)) {+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);+ return false;+ }++ if (isnan_fp16(f)) {+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);+ return false;+ }++ return true;+}++#define VALIDATE_ROW_DATA_D_F16_IMPL(type, data, nb) \+ const type * q = (const type *) (data); \+ for (size_t i = 0; i < (nb); ++i) { \+ if (!validate_fp16(q[i].d, i)) { \+ return false; \+ } \+ }++#define VALIDATE_ROW_DATA_DM_F16_IMPL(type, data, nb, d, m) \+ const type * q = (const type *) (data); \+ for (size_t i = 0; i < (nb); ++i) { \+ if (!validate_fp16(q[i].d, i) || !validate_fp16(q[i].m, i)) { \+ return false; \+ } \+ }++bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) {+ if (type < 0 || type >= GGML_TYPE_COUNT) {+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);+ return false;+ }++ if (nbytes % ggml_type_size(type) != 0) {+ fprintf(stderr, "%s: invalid size %zu for type %d\n", __func__, nbytes, type);+ return false;+ }++ const size_t nb = nbytes/ggml_type_size(type);++ switch (type) {+ case GGML_TYPE_BF16:+ {+ int nans = 0;+ int infs = 0;+ const unsigned short * f = (const unsigned short *) data;+ for (size_t i = 0; i < nb; ++i) {+ nans += (f[i] & 0x7fff) > 0x7f80;+ infs += (f[i] & 0x7fff) == 0x7f80;+ }+ if (nans) {+ fprintf(stderr, "%s: found %d NaNs in row of %zu BF16 values\n", __func__, nans, nb);+ return false;+ }+ if (infs) {+ fprintf(stderr, "%s: found %d infinities in row of %zu BF16 values\n", __func__, infs, nb);+ return false;+ }+ } break;+ case GGML_TYPE_F16:+ {+ const ggml_fp16_t * f = (const ggml_fp16_t *) data;+ size_t i = 0;+#if defined(__AVX2__)+ for (; i + 15 < nb; i += 16) {+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi16(0x7c00));+ __m256i cmp = _mm256_cmpeq_epi16(vexp, _mm256_set1_epi16(0x7c00));+ int mask = _mm256_movemask_epi8(cmp);+ if (mask) {+ for (size_t j = 0; j < 16; ++j) {+ if (!validate_fp16(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#elif defined(__ARM_NEON)+ for (; i + 7 < nb; i += 8) {+ uint16x8_t v = vld1q_u16(f + i);+ uint16x8_t vexp = vandq_u16(v, vdupq_n_u16(0x7c00));+ uint16x8_t cmp = vceqq_u16(vexp, vdupq_n_u16(0x7c00));+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(cmp, 4)), 0);+ if (mask) {+ for (size_t j = 0; j < 8; ++j) {+ if (!validate_fp16(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#endif+ for (; i < nb; ++i) {+ if (!validate_fp16(f[i], i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_F32:+ {+ const float * f = (const float *) data;+ size_t i = 0;+#if defined(__AVX2__)+ for (; i + 7 < nb; i += 8) {+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi32(0x7f800000));+ __m256i cmp = _mm256_cmpeq_epi32(vexp, _mm256_set1_epi32(0x7f800000));+ int mask = _mm256_movemask_epi8(cmp);+ if (mask) {+ for (size_t j = 0; j < 8; ++j) {+ if (!validate_float(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#elif defined(__ARM_NEON)+ for (; i + 3 < nb; i += 4) {+ uint32x4_t v = vld1q_u32((const uint32_t *)f + i);+ uint32x4_t vexp = vandq_u32(v, vdupq_n_u32(0x7f800000));+ uint32x4_t cmp = vceqq_u32(vexp, vdupq_n_u32(0x7f800000));+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u16(vshrn_n_u32(cmp, 8)), 0);+ if (mask) {+ for (size_t j = 0; j < 4; ++j) {+ if (!validate_float(f[i + j], i + j)) {+ return false;+ }+ }+ GGML_UNREACHABLE();+ }+ }+#endif+ for (; i < nb; ++i) {+ if (!validate_float(f[i], i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_F64:+ {+ const double * f = (const double *) data;+ for (size_t i = 0; i < nb; ++i) {+ if (!validate_float(f[i], i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_Q4_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb);+ } break;+ case GGML_TYPE_Q4_1:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_1, data, nb, d, m);+ } break;+ case GGML_TYPE_Q5_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_0, data, nb);+ } break;+ case GGML_TYPE_Q5_1:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_1, data, nb, d, m);+ } break;+ case GGML_TYPE_Q8_0:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q8_0, data, nb);+ } break;+ case GGML_TYPE_Q2_K:+ {+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin);+ } break;+ case GGML_TYPE_Q3_K:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_K, data, nb);+ } break;+ case GGML_TYPE_Q4_K:+ {+ #ifdef GGML_QKK_64+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d[0], d[1]);+ #else+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d, dmin);+ #endif+ } break;+ case GGML_TYPE_Q5_K:+ {+ #ifdef GGML_QKK_64+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_K, data, nb);+ #else+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_K, data, nb, d, dmin);+ #endif+ } break;+ case GGML_TYPE_Q6_K:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q6_K, data, nb);+ } break;+ case GGML_TYPE_Q8_K:+ {+ const block_q8_K * q = (const block_q8_K *) data;+ for (size_t i = 0; i < nb; ++i) {+ if (!validate_float(q[i].d, i)) {+ return false;+ }+ }+ } break;+ case GGML_TYPE_IQ1_S:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb);+ } break;+ case GGML_TYPE_IQ1_M:+ {+ const block_iq1_m * q = (const block_iq1_m *) data;+ for (size_t i = 0; i < nb; ++i) {+ #if QK_K == 64+ if (!validate_fp16(q[i].d, i)) {+ return false;+ }+ #else+ iq1m_scale_t scale;+ const uint16_t * sc = (const uint16_t *)q[i].scales;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);+ if (!validate_fp16(scale.f16, i)) {+ return false;+ }+ #endif+ }+ } break;+ case GGML_TYPE_IQ2_XXS:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xxs, data, nb);+ } break;+ case GGML_TYPE_IQ2_XS:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xs, data, nb);+ } break;+ case GGML_TYPE_IQ2_S:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_s, data, nb);+ } break;+ case GGML_TYPE_IQ3_XXS:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_xxs, data, nb);+ } break;++ case GGML_TYPE_IQ3_S:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_s, data, nb);+ } break;+ case GGML_TYPE_IQ4_XS:+ #if QK_K != 64+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_xs, data, nb);+ } break;+ #endif+ // with QK_K == 64, iq4_xs is iq4_nl+ case GGML_TYPE_IQ4_NL:+ {+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);+ } break;+ case GGML_TYPE_I8:+ case GGML_TYPE_I16:+ case GGML_TYPE_I32:+ case GGML_TYPE_I64:+ // nothing to validate+ break;+ default:+ {+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);+ return false;+ }+ }++ return true;+}diff --git a/llama.cpp/ggml-quants.py b/llama.cpp/ggml-quants.pynew file mode 100644index 00000000..c3d0a469--- /dev/null+++ b/llama.cpp/ggml-quants.py@@ -0,0 +1,77 @@+import re++with open('llama.cpp/ggml-quants.h') as f:+ prototypes = f.read()+prototypes = [line.replace(';', '') for line in prototypes.split('\n')+ if line.endswith(';') and not line.startswith('//')]+FUNCS = [(re.search(r'(?<= )\w+', proto).group(0), proto)+ for proto in prototypes]++# BEGIN SPECIAL FUNCTIONS+FUNCS.append(('ggml_validate_row_data', 'bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes)'))+# END SPECIAL FUNCTIONS++ARCHS = (+ ('amd_avx512', '__x86_64__', ('FMA', 'F16C', 'AVX2', 'AVX512F')),+ ('amd_avx2', '__x86_64__', ('FMA', 'F16C', 'AVX2')),+ ('amd_avx', '__x86_64__', ()),+ ('arm80', '__aarch64__', ()),+)++for arch, mac, needs in ARCHS:+ path = 'llama.cpp/ggml-quants-%s.c' % (arch.replace('_', '-'))+ with open(path, 'w') as f:+ f.write('#ifdef %s\n' % (mac))+ for func, proto in FUNCS:+ f.write('#define %s %s_%s\n' % (func, func, arch))+ f.write('#include "ggml-quants.inc"\n')+ f.write('#endif // %s\n' % (mac))++with open('llama.cpp/ggml-quants.cpp', 'w') as f:+ f.write('#include <cosmo.h>\n')+ f.write('#include <sys/auxv.h>\n')+ f.write('#include <libc/sysv/consts/hwcap.h>\n')+ f.write('#include "ggml-quants.h"\n')+ f.write('\n')+ for func, proto in FUNCS:+ for arch, mac, needs in ARCHS:+ f.write('extern "C" %s;\n' % (proto.replace(func, '%s_%s' % (func, arch))))+ f.write('\n')+ f.write('static const struct QuantFuncs {\n')+ for func, proto in FUNCS:+ f.write(' typeof(%s) *ptr_%s;\n' % (func, func))+ f.write('\n')+ f.write(' QuantFuncs() {\n')+ f.write('#ifdef __x86_64__\n')+ for arch, mac, needs in ARCHS:+ if mac == '__x86_64__':+ f.write(' if (%s) {\n' % (' && '.join('X86_HAVE(%s)' % (need) for need in needs) or '1'))+ for func, proto in FUNCS:+ f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))+ f.write(' return;\n')+ f.write(' }\n')+ f.write('#else\n')+ for func, proto in FUNCS:+ f.write(' ptr_%s = %s_arm80;\n' % (func, func))+ f.write('#endif\n')+ f.write(' }\n')+ f.write('} funcs;\n')+ f.write('\n')+ for func, proto in FUNCS:+ proto = proto.replace(';', '')+ f.write(proto + ' {\n')+ if 'imatrix' in proto:+ args = 'src, dst, nrows, n_per_row, imatrix'+ elif 'quantize' in proto:+ args = 'x, y, k'+ elif 'vec_dot' in proto:+ args = 'n, s, bs, vx, bx, vy, by, nrc'+ elif 'grid_size' in proto:+ args = 'grid_size'+ elif 'validate' in proto:+ args = 'type, data, nbytes'+ else:+ args = 'type'+ f.write(' return funcs.ptr_%s(%s);\n' % (func, args))+ f.write('}\n')+ f.write('\n')diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 37715f20..371f738a 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -117,6 +117,7 @@ void ggml_print_backtrace(void) {/*#define GGML_PERF*/#define GGML_DEBUG 0+#define GGML_KAHAN 0#define GGML_SOFT_MAX_UNROLL 4#define GGML_VEC_DOT_UNROLL 2@@ -956,7 +957,7 @@ do { \#define GGML_F16_VEC_ZERO GGML_F16x8_ZERO#define GGML_F16_VEC_SET1 GGML_F16x8_SET1#define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)- #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE(p, r[i])+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((ggml_fp16_internal_t *)(p), r[i])#define GGML_F16_VEC_FMA GGML_F16x8_FMA#define GGML_F16_VEC_ADD GGML_F16x8_ADD#define GGML_F16_VEC_SUB GGML_F16x8_SUB@@ -984,7 +985,7 @@ do { \#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)- #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((ggml_fp16_internal_t *)(p), r[i])#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD#define GGML_F16_VEC_SUB GGML_F32Cx4_SUB@@ -1056,7 +1057,7 @@ do { \// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F// so F16C guard isn't required-#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(x)))+#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)@@ -1140,10 +1141,10 @@ do { \#if defined(__F16C__)// the _mm256_cvt intrinsics require F16C-#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(x)))+#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))#else-static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {+static inline __m256 __avx_f32cx8_load(const ggml_fp16_t *x) {float tmp[8];for (int i = 0; i < 8; i++) {@@ -1613,10 +1614,14 @@ static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t *ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[j]);- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[j], ky);- err[j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[j]), ky);- sum[j] = kt;+ if (GGML_KAHAN) {+ GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[j]);+ GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[j], ky);+ err[j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[j]), ky);+ sum[j] = kt;+ } else {+ sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);+ }}}@@ -1663,10 +1668,14 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * refor (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[k][j]);- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[k][j], ky);- err[k][j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[k][j]), ky);- sum[k][j] = kt;+ if (GGML_KAHAN) {+ GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[k][j]);+ GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[k][j], ky);+ err[k][j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[k][j]), ky);+ sum[k][j] = kt;+ } else {+ sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);+ }}}}@@ -1726,6 +1735,37 @@ inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float#endif}+inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {+#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);++ GGML_F16_VEC ax[GGML_F16_ARR];+ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);++ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);+ }+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);+ }+#endif+}+// xs and vs are byte strides of x and vinline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {@@ -1810,6 +1850,35 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {#endif}+inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {+#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);++ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_MUL(ay[j], vx);++ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);+ }+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);+ }+#endif+}+inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }@@ -1828,39 +1897,6 @@ inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float////////////////////////////////////////////////////////////////////////////////// SYNCHRONIZATION PRIMITIVES-struct ggml_barrier {- int nth;- atomic_uint phase;- atomic_uint count;-};--int ggml_delay(int backoff) {- if (backoff < 13) {- volatile int i;- for (i = 0; i != 1 << backoff; i++) {- }- backoff++;- } else {- sched_yield();- }- return backoff;-}--// creates barrier and blocks until all threads call this-void ggml_syncthreads(struct ggml_compute_params * params) {- struct ggml_barrier * b = params->barrier;- unsigned phase = atomic_load_explicit(&b->phase, memory_order_relaxed);- if (atomic_fetch_add_explicit(&b->count, 1, memory_order_acq_rel) + 1 == b->nth) {- atomic_store_explicit(&b->count, 0, memory_order_relaxed);- atomic_store_explicit(&b->phase, phase + 1, memory_order_release);- } else {- int backoff = 0;- while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase) {- backoff = ggml_delay(backoff);- }- }-}-void ggml_once(atomic_uint * once, void init(void)) {uint32_t old = atomic_load_explicit(once, memory_order_acquire);if (!old && atomic_compare_exchange_strong_explicit(once, &old, 1,@@ -2104,6 +2140,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"LEAKY_RELU","FLASH_ATTN",+ "FLASH_ATTN_EXT","FLASH_FF","FLASH_ATTN_BACK","SSM_CONV",@@ -2130,7 +2167,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"CROSS_ENTROPY_LOSS_BACK",};-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");+static_assert(GGML_OP_COUNT == 77, "GGML_OP_COUNT != 77");static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"none",@@ -2194,6 +2231,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"leaky_relu(x)","flash_attn(x)",+ "flash_attn_ext(x)","flash_ff(x)","flash_attn_back(x)","ssm_conv(x)",@@ -2220,7 +2258,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"cross_entropy_loss_back(x,y)",};-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");+static_assert(GGML_OP_COUNT == 77, "GGML_OP_COUNT != 77");static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");@@ -2716,7 +2754,7 @@ GGML_CALL bool ggml_is_empty(const struct ggml_tensor * tensor) {return false;}-bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {+GGML_CALL bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");return@@ -4668,6 +4706,8 @@ struct ggml_tensor * ggml_mul_mat(void ggml_mul_mat_set_prec(struct ggml_tensor * a,enum ggml_prec prec) {+ GGML_ASSERT(a->op == GGML_OP_MUL_MAT);+const int32_t prec_i32 = (int32_t) prec;ggml_set_op_params_i32(a, 0, prec_i32);@@ -5506,17 +5546,23 @@ static struct ggml_tensor * ggml_soft_max_impl(GGML_ASSERT(ggml_is_contiguous(a));if (mask) {+ GGML_ASSERT(mask->type == GGML_TYPE_F16 || mask->type == GGML_TYPE_F32);GGML_ASSERT(ggml_is_contiguous(mask));GGML_ASSERT(ggml_is_matrix(mask));- GGML_ASSERT(ggml_can_repeat_rows(mask, a));+ GGML_ASSERT(mask->ne[0] == a->ne[0]);+ GGML_ASSERT(mask->ne[1] >= a->ne[1]);}if (pos) {GGML_ASSERT(ggml_is_vector(pos));- GGML_ASSERT(pos->type == GGML_TYPE_F32);+ GGML_ASSERT(pos->type == GGML_TYPE_F16 || pos->type == GGML_TYPE_F32);GGML_ASSERT(pos->ne[0] == a->ne[0]);}+ if (pos && mask) {+ GGML_ASSERT(pos->type == mask->type);+ }+if (max_bias > 0.0f) {GGML_ASSERT(pos);}@@ -6325,6 +6371,59 @@ struct ggml_tensor * ggml_flash_attn(return result;}+// ggml_flash_attn_ext++struct ggml_tensor * ggml_flash_attn_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * q,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * mask,+ float scale) {+ GGML_ASSERT(ggml_can_mul_mat(k, q));+ // TODO: check if vT can be multiplied by (k*qT)+ if (mask) {+ GGML_ASSERT(ggml_is_contiguous(mask));+ GGML_ASSERT(mask->ne[2] == 1);+ GGML_ASSERT(mask->ne[3] == 1);+ GGML_ASSERT(mask->ne[1] >= GGML_PAD(q->ne[1], GGML_KQ_MASK_PAD) &&+ "the Flash-Attention kernel requires the mask to be padded to GGML_KQ_MASK_PAD and at least n_queries big");+ //GGML_ASSERT(ggml_can_repeat_rows(mask, qk));+ }++ bool is_node = false;++ if (q->grad || k->grad || v->grad) {+ is_node = true;+ }++ // permute(0, 2, 1, 3)+ int64_t ne[4] = { q->ne[0], q->ne[2], q->ne[1], q->ne[3] };+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);++ float params[] = { scale };+ ggml_set_op_params(result, params, sizeof(params));++ result->op = GGML_OP_FLASH_ATTN_EXT;+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;+ result->src[0] = q;+ result->src[1] = k;+ result->src[2] = v;+ result->src[3] = mask;++ return result;+}++void ggml_flash_attn_ext_set_prec(+ struct ggml_tensor * a,+ enum ggml_prec prec) {+ GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT);++ const int32_t prec_i32 = (int32_t) prec;++ ggml_set_op_params_i32(a, 1, prec_i32); // scale is on first pos+}+// ggml_flash_ffstruct ggml_tensor * ggml_flash_ff(@@ -9609,8 +9708,8 @@ static void ggml_compute_forward_concat_f32(}static void ggml_compute_forward_concat(- struct ggml_compute_params * params,- struct ggml_tensor * dst) {+ struct ggml_compute_params* params,+ struct ggml_tensor* dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10936,7 +11035,8 @@ static void ggml_compute_forward_mul_mat(params->type,src0->type,src1->type,- dst->type))+ dst->type,+ dst->op_params[0]))goto UseGgmlGemm1;return;}@@ -10951,11 +11051,11 @@ UseGgmlGemm1:;assert(params->wsize >= ne11*ne12*ne13*row_size);GGML_ASSERT(src1->type == GGML_TYPE_F32);- unsigned chore = 0;+ unsigned chore = 0; // [jart] need for speedfor (int64_t i13 = 0; i13 < ne13; ++i13) {for (int64_t i12 = 0; i12 < ne12; ++i12) {for (int64_t i11 = 0; i11 < ne11; ++i11) {- if (chore++ % nth == ith) {+ if (chore++ % nth == ith) { // [jart] need for speedfrom_float_to_vec_dot((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), (void *) wdata, ne10);}wdata += row_size;@@ -10975,15 +11075,13 @@ UseGgmlGemm1:;const size_t row_size = ggml_row_size(vec_dot_type, ne10);#if GGML_USE_LLAMAFILE- if (src1_cont) {+ if (src1->type != vec_dot_type) {for (int64_t i13 = 0; i13 < ne13; i13++)for (int64_t i12 = 0; i12 < ne12; i12++)if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type),(const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,nb01/ggml_type_size(src0->type),- (const char *)wdata + ggml_row_size(vec_dot_type,- nb12/ggml_type_size(src1->type)*i12 +- nb13/ggml_type_size(src1->type)*i13),+ (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,row_size/ggml_type_size(vec_dot_type),(char *)dst->data + i12*nb2 + i13*nb3,nb1/ggml_type_size(dst->type),@@ -10991,7 +11089,8 @@ UseGgmlGemm1:;params->type,src0->type,vec_dot_type,- dst->type))+ dst->type,+ dst->op_params[0]))goto UseGgmlGemm2;return;}@@ -11092,11 +11191,11 @@ UseGgmlGemm2:;}#if 0- fprintf(stderr, "MULMAT %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) * %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) -> %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) vdt=%s\n",+ fprintf(stderr, "MULMAT %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) * %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) -> %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) vdt=%s src1_cont=%d\n",ggml_type_name(src0->type), ne00, ne01, ne02, nb00, nb01, nb02,ggml_type_name(src1->type), ne10, ne11, ne12, nb10, nb11, nb12,ggml_type_name(src0->type), ne0, ne1, ne2, nb0, nb1, nb2,- ggml_type_name(vec_dot_type));+ ggml_type_name(vec_dot_type), src1_cont);#endif}@@ -11113,11 +11212,11 @@ static void ggml_compute_forward_mul_mat_id(if (llamafile_mixmul(params, src0, src1, ids, dst))return;+ GGML_TENSOR_BINARY_OP_LOCALS+const int ith = params->ith;const int nth = params->nth;- GGML_TENSOR_BINARY_OP_LOCALS-const enum ggml_type type = src0->type;const bool src1_cont = ggml_is_contiguous(src1);@@ -12424,7 +12523,7 @@ static void ggml_compute_forward_soft_max_f32(GGML_TENSOR_UNARY_OP_LOCALS- const int64_t ne11 = src1 ? src1->ne[1] : 1;+ //const int64_t ne11 = src1 ? src1->ne[1] : 1;// TODO: is this supposed to be ceil instead of floor?// https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370@@ -12447,19 +12546,31 @@ static void ggml_compute_forward_soft_max_f32(float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;// when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching- float * pos = src2 ? (float *) src2->data : src0->data;+ ggml_fp16_t * pos_f16 = src2 ? (ggml_fp16_t *) src2->data : src0->data;+ float * pos_f32 = src2 ? (float *) src2->data : src0->data;++ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);for (int i1 = ir0; i1 < ir1; i1++) {float * sp = (float *)((char *) src0->data + i1*src0->nb[1]);float * dp = (float *)((char *) dst->data + i1*dst->nb[1]);// broadcast the mask across rows- float * mp = src1 ? (float *)((char *) src1->data + (i1%ne11)*src1->nb[1]) : NULL;+ ggml_fp16_t * mp_f16 = src1 ? (ggml_fp16_t *)((char *) src1->data) + (i1%ne01)*ne00 : NULL;+ float * mp_f32 = src1 ? (float *)((char *) src1->data) + (i1%ne01)*ne00 : NULL;ggml_vec_cpy_f32 (nc, wp, sp);ggml_vec_scale_f32(nc, wp, scale);- if (mp) {- ggml_vec_acc_f32(nc, wp, mp);+ if (mp_f32) {+ if (use_f16) {+ for (int i = 0; i < nc; ++i) {+ wp[i] += GGML_FP16_TO_FP32(mp_f16[i]);+ }+ } else {+ for (int i = 0; i < nc; ++i) {+ wp[i] += mp_f32[i];+ }+ }}// ALiBi bias@@ -12467,8 +12578,14 @@ static void ggml_compute_forward_soft_max_f32(const uint32_t h = (i1/ne01)%ne02; // headconst float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1);- for (int i = 0; i < nc; i++) {- wp[i] = wp[i] + slope*pos[i];+ if (use_f16) {+ for (int i = 0; i < nc; ++i) {+ wp[i] += slope*GGML_FP16_TO_FP32(pos_f16[i]);+ }+ } else {+ for (int i = 0; i < nc; ++i) {+ wp[i] += slope*pos_f32[i];+ }}}@@ -14715,6 +14832,198 @@ static void ggml_compute_forward_flash_attn(}}+// ggml_compute_forward_flash_attn_ext++static void ggml_compute_forward_flash_attn_ext_f16(+ struct ggml_compute_params * params,+ const struct ggml_tensor * q,+ const struct ggml_tensor * k,+ const struct ggml_tensor * v,+ const struct ggml_tensor * mask,+ struct ggml_tensor * dst) {+ int64_t t0 = ggml_perf_time_us();+ UNUSED(t0);++ GGML_TENSOR_LOCALS(int64_t, neq, q, ne)+ GGML_TENSOR_LOCALS(size_t, nbq, q, nb)+ GGML_TENSOR_LOCALS(int64_t, nek, k, ne)+ GGML_TENSOR_LOCALS(size_t, nbk, k, nb)+ GGML_TENSOR_LOCALS(int64_t, nev, v, ne)+ GGML_TENSOR_LOCALS(size_t, nbv, v, nb)+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)++ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t D = neq0;+ const int64_t N = neq1;++ GGML_ASSERT(ne0 == D);+ GGML_ASSERT(ne2 == N);++ GGML_ASSERT(nbq0 == sizeof(float));+ GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));++ GGML_ASSERT(neq0 == D);+ GGML_ASSERT(nek0 == D);+ GGML_ASSERT(nev0 == D);++ GGML_ASSERT(neq1 == N);+ GGML_ASSERT(nev0 == D);++ // dst cannot be transposed or permuted+ GGML_ASSERT(nb0 == sizeof(float));+ GGML_ASSERT(nb0 <= nb1);+ GGML_ASSERT(nb1 <= nb2);+ GGML_ASSERT(nb2 <= nb3);++ // broadcast factors+ const int64_t rk2 = neq2/nek2;+ const int64_t rk3 = neq3/nek3;++ const int64_t rv2 = neq2/nev2;+ const int64_t rv3 = neq3/nev3;++ if (params->type == GGML_TASK_TYPE_INIT) {+ return;+ }++ if (params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // parallelize by q rows using ggml_vec_dot_f32++ // total rows in q+ const int nr = neq1*neq2*neq3;++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ float scale = 1.0f;+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));++ // loop over n_batch and n_head+ for (int ir = ir0; ir < ir1; ++ir) {+ // q indices+ const int iq3 = ir/(neq2*neq1);+ const int iq2 = (ir - iq3*neq2*neq1)/neq1;+ const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);++ float S = 0.0f;+ float M = -INFINITY;++ float * V32 = (float *) params->wdata + ith*(2*D + CACHE_LINE_SIZE_F32);+ ggml_fp16_t * Q16 = (ggml_fp16_t *) (V32); // reuse memory+ ggml_fp16_t * V16 = (ggml_fp16_t *) (V32 + D);++ memset(V16, 0, D*sizeof(ggml_fp16_t));++ const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1]) : NULL;++ // k indices+ const int ik3 = iq3 / rk3;+ const int ik2 = iq2 / rk2;++ // v indices+ const int iv3 = iq3 / rv3;+ const int iv2 = iq2 / rv2;++ // online softmax / attention+ // loop over n_kv and n_head_kv+ // ref: https://arxiv.org/pdf/2112.05682.pdf+ for (int64_t ic = 0; ic < nek1; ++ic) {+ const float mv = mp ? GGML_FP16_TO_FP32(mp[ic]) : 0.0f;+ if (mv == -INFINITY) {+ continue;+ }++ float s;++ // convert Q to F16 in V32+ {+ const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));++ for (int64_t d = 0; d < D; ++d) {+ Q16[d] = GGML_FP32_TO_FP16(pq[d]);+ }+ }++ ggml_vec_dot_f16(D,+ &s, 0,+ (ggml_fp16_t *) ((char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,+ Q16, 0, 1);++ s = s*scale + mv;++ const float Mold = M;++ float ms = 1.0f;+ float vs = 1.0f;++ if (s > M) {+ M = s;+ ms = expf(Mold - M);++ // V = V*expf(Mold - M)+ ggml_vec_scale_f16(D, V16, ms);+ } else {+ vs = expf(s - M);+ }++ const ggml_fp16_t * v16 = (const ggml_fp16_t *) ((char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));++ // V += v*expf(s - M)+ ggml_vec_mad_f16(D, V16, v16, vs);++ S = S*ms + vs;+ }++ // V /= S+ for (int64_t d = 0; d < D; ++d) {+ V32[d] = GGML_FP16_TO_FP32(V16[d])/S;+ }++ // dst indices+ const int i1 = iq1;+ const int i2 = iq2;+ const int i3 = iq3;++ // original+ //memcpy((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3), V, nev0*sizeof(float));++ // permute(0, 2, 1, 3)+ memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, V32, nb1);+ }+}++static void ggml_compute_forward_flash_attn_ext(+ struct ggml_compute_params * params,+ const struct ggml_tensor * q,+ const struct ggml_tensor * k,+ const struct ggml_tensor * v,+ const struct ggml_tensor * mask,+ struct ggml_tensor * dst) {+ switch (dst->op_params[1]) {+ case GGML_PREC_DEFAULT:+ case GGML_PREC_F32:+ {+ // uses F32 accumulators+ ggml_compute_forward_flash_attn_ext_f16(params, q, k, v, mask, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}+// ggml_compute_forward_flash_ffstatic void ggml_compute_forward_flash_ff_f16(@@ -16494,6 +16803,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggmconst bool masked = t != 0;ggml_compute_forward_flash_attn(params, masked, tensor);} break;+ case GGML_OP_FLASH_ATTN_EXT:+ {+ ggml_compute_forward_flash_attn_ext(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], tensor);+ } break;case GGML_OP_FLASH_FF:{ggml_compute_forward_flash_ff(params, tensor);@@ -17506,6 +17819,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorGGML_ASSERT(false); // TODO: not implemented} break;case GGML_OP_FLASH_ATTN:+ case GGML_OP_FLASH_ATTN_EXT:{struct ggml_tensor * flash_grad = NULL;if (src0->grad || src1->grad || tensor->src[2]->grad) {@@ -18278,6 +18592,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_n_tasks = n_threads;} break;case GGML_OP_FLASH_ATTN:+ case GGML_OP_FLASH_ATTN_EXT:{n_tasks = n_threads;} break;@@ -18702,6 +19017,12 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threacur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2}} break;+ case GGML_OP_FLASH_ATTN_EXT:+ {+ const int64_t ne00 = node->src[0]->ne[0]; // D++ cur = 2*sizeof(float)*ne00*n_tasks; // 2x head size+ } break;case GGML_OP_FLASH_FF:{if (node->src[1]->type == GGML_TYPE_F32) {@@ -20775,7 +21096,7 @@ static void gguf_free_kv(struct gguf_kv * kv) {}struct gguf_context * gguf_init_empty(void) {- struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));+ struct gguf_context * ctx = GGML_CALLOC(1, sizeof(struct gguf_context));memcpy(ctx->header.magic, GGUF_MAGIC, sizeof(ctx->header.magic));ctx->header.version = GGUF_VERSION;@@ -20815,7 +21136,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_ibool ok = true;- struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));+ struct gguf_context * ctx = GGML_CALLOC(1, sizeof(struct gguf_context));// read the header{@@ -20850,9 +21171,13 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i// read the kv pairs{- ctx->kv = GGML_MALLOC(ctx->header.n_kv * sizeof(struct gguf_kv));+ const uint64_t n_kv = ctx->header.n_kv;- for (uint64_t i = 0; i < ctx->header.n_kv; ++i) {+ // header.n_kv will hold the actual value of pairs that were successfully read in the loop below+ ctx->header.n_kv = 0;+ ctx->kv = GGML_CALLOC(n_kv, sizeof(struct gguf_kv));++ for (uint64_t i = 0; i < n_kv; ++i) {struct gguf_kv * kv = &ctx->kv[i];//fprintf(stderr, "%s: reading kv %d\n", __func__, i);@@ -20900,7 +21225,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_ireturn NULL;}- kv->value.arr.data = GGML_MALLOC(kv->value.arr.n * gguf_type_size(kv->value.arr.type));+ kv->value.arr.data = GGML_CALLOC(kv->value.arr.n, gguf_type_size(kv->value.arr.type));ok = ok && gguf_fread_el(file, kv->value.arr.data, kv->value.arr.n * gguf_type_size(kv->value.arr.type), &offset);} break;@@ -20913,7 +21238,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_ireturn NULL;}- kv->value.arr.data = GGML_MALLOC(kv->value.arr.n * sizeof(struct gguf_str));+ kv->value.arr.data = GGML_CALLOC(kv->value.arr.n, sizeof(struct gguf_str));for (uint64_t j = 0; j < kv->value.arr.n; ++j) {ok = ok && gguf_fread_str(file, &((struct gguf_str *) kv->value.arr.data)[j], &offset);@@ -20929,6 +21254,8 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_iif (!ok) {break;}++ ctx->header.n_kv++;}if (!ok) {@@ -20940,7 +21267,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i// read the tensor infos{- ctx->infos = GGML_MALLOC(ctx->header.n_tensors * sizeof(struct gguf_tensor_info));+ ctx->infos = GGML_CALLOC(ctx->header.n_tensors, sizeof(struct gguf_tensor_info));for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {struct gguf_tensor_info * info = &ctx->infos[i];@@ -20961,8 +21288,17 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_iok = ok && gguf_fread_el (file, &info->type, sizeof(info->type), &offset);ok = ok && gguf_fread_el (file, &info->offset, sizeof(info->offset), &offset);+ // TODO: return an error instead of crashing with GGML_ASSERTgguf_tensor_info_sanitize(info);+ // make sure there is no duplicated tensor names+ for (uint64_t j = 0; j < i; ++j) {+ if (strcmp(info->name.data, ctx->infos[j].name.data) == 0) {+ fprintf(stderr, "%s: duplicated tensor name %s\n", __func__, info->name.data);+ ok = false;+ }+ }+if (!ok) {fprintf(stderr, "%s: failed to read tensor info\n", __func__);gguf_free(ctx);@@ -21125,7 +21461,7 @@ void gguf_free(struct gguf_context * ctx) {GGML_FREE(ctx->infos);}- GGML_ALIGNED_FREE(ctx);+ GGML_FREE(ctx);}const char * gguf_type_name(enum gguf_type type) {@@ -21436,7 +21772,7 @@ void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_tyctx->kv[idx].type = GGUF_TYPE_ARRAY;ctx->kv[idx].value.arr.type = type;ctx->kv[idx].value.arr.n = n;- ctx->kv[idx].value.arr.data = GGML_MALLOC(n*gguf_type_size(type));+ ctx->kv[idx].value.arr.data = GGML_CALLOC(n, gguf_type_size(type));memcpy(ctx->kv[idx].value.arr.data, data, n*gguf_type_size(type));}@@ -21446,7 +21782,7 @@ void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char **ctx->kv[idx].type = GGUF_TYPE_ARRAY;ctx->kv[idx].value.arr.type = GGUF_TYPE_STRING;ctx->kv[idx].value.arr.n = n;- ctx->kv[idx].value.arr.data = GGML_MALLOC(n*sizeof(struct gguf_str));+ ctx->kv[idx].value.arr.data = GGML_CALLOC(n, sizeof(struct gguf_str));for (int i = 0; i < n; i++) {struct gguf_str * str = &((struct gguf_str *)ctx->kv[idx].value.arr.data)[i];str->n = strlen(data[i]);@@ -21473,7 +21809,7 @@ void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {case GGUF_TYPE_ARRAY:{if (src->kv[i].value.arr.type == GGUF_TYPE_STRING) {- const char ** data = GGML_MALLOC(src->kv[i].value.arr.n*sizeof(char *));+ const char ** data = GGML_CALLOC(src->kv[i].value.arr.n, sizeof(char *));for (uint32_t j = 0; j < src->kv[i].value.arr.n; j++) {data[j] = ((struct gguf_str *)src->kv[i].value.arr.data)[j].data;}@@ -21493,6 +21829,10 @@ void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {void gguf_add_tensor(struct gguf_context * ctx,const struct ggml_tensor * tensor) {+ if (gguf_find_tensor(ctx, tensor->name) != -1) {+ GGML_ASSERT(false && "duplicated tensor name");+ }+const int idx = ctx->header.n_tensors;ctx->infos = realloc(ctx->infos, (idx + 1)*sizeof(struct gguf_tensor_info));@@ -21561,7 +21901,7 @@ struct gguf_buf {static struct gguf_buf gguf_buf_init(size_t size) {struct gguf_buf buf = {- /*buf.data =*/ size == 0 ? NULL : GGML_MALLOC(size),+ /*buf.data =*/ size == 0 ? NULL : GGML_CALLOC(1, size),/*buf.size =*/ size,/*buf.offset =*/ 0,};diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex d9aa1eb8..b45c29ba 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -479,6 +479,7 @@ extern "C" {GGML_OP_LEAKY_RELU,GGML_OP_FLASH_ATTN,+ GGML_OP_FLASH_ATTN_EXT,GGML_OP_FLASH_FF,GGML_OP_FLASH_ATTN_BACK,GGML_OP_SSM_CONV,@@ -683,14 +684,8 @@ extern "C" {// work buffer for all threadssize_t wsize;void * wdata;-- struct ggml_barrier * barrier;};- GGML_API void ggml_syncthreads (struct ggml_compute_params *);- GGML_API void * ggml_acquire_wdata(struct ggml_compute_params *);- GGML_API void ggml_release_wdata(struct ggml_compute_params *);-// numa strategiesenum ggml_numa_strategy {GGML_NUMA_STRATEGY_DISABLED = 0,@@ -767,11 +762,13 @@ extern "C" {GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars- GGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);+ GGML_API GGML_CALL bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);// use this to compute the memory overhead of a tensorGGML_API size_t ggml_tensor_overhead(void);+ GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);+// mainGGML_API struct ggml_context * ggml_init(struct ggml_init_params params);@@ -1730,6 +1727,25 @@ extern "C" {struct ggml_tensor * v,bool masked);+#define GGML_KQ_MASK_PAD 32++ // q: [n_embd, n_batch, n_head, 1]+ // k: [n_embd, n_kv, n_head_kv, 1]+ // v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!+ // mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = GGML_PAD(n_batch, GGML_KQ_MASK_PAD) !!+ // res: [n_embd, n_head, n_batch, 1] !! permuted !!+ GGML_API struct ggml_tensor * ggml_flash_attn_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * q,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * mask,+ float scale);++ GGML_API void ggml_flash_attn_ext_set_prec(+ struct ggml_tensor * a,+ enum ggml_prec prec);+GGML_API struct ggml_tensor * ggml_flash_attn_back(struct ggml_context * ctx,struct ggml_tensor * q,diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.incdeleted file mode 100644index 3c675205..00000000--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc+++ /dev/null@@ -1,522 +0,0 @@-static void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- assert(n % QK_K == 0);- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);-- const uint32_t kmask1 = 0x03030303;- const uint32_t kmask2 = 0x0f0f0f0f;-- const block_q3_K * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;--#ifdef __ARM_NEON-- uint32_t aux[3];- uint32_t utmp[4];-- const uint8x16_t m3b = vdupq_n_u8(0x3);- const int32x4_t vzero = vdupq_n_s32(0);-- const uint8x16_t m0 = vdupq_n_u8(1);- const uint8x16_t m1 = vshlq_n_u8(m0, 1);- const uint8x16_t m2 = vshlq_n_u8(m0, 2);- const uint8x16_t m3 = vshlq_n_u8(m0, 3);- const int8_t m32 = 32;-- ggml_int8x16x4_t q3bytes;-- float sum = 0;-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q3 = x[i].qs;- const uint8_t * restrict qh = x[i].hmask;- const int8_t * restrict q8 = y[i].qs;-- ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);-- ggml_uint8x16x4_t q3h;-- int32_t isum = 0;-- // Set up scales- memcpy(aux, x[i].scales, 12);- utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);- utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);- utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);- utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);-- int8_t * scale = (int8_t *)utmp;- for (int j = 0; j < 16; ++j) scale[j] -= m32;-- for (int j = 0; j < QK_K/128; ++j) {-- const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;- const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;- const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;-- q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);- q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);- q3h.val[2] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[0]), 1);- q3h.val[3] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[1]), 1);-- q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[0], m3b)), vreinterpretq_s8_u8(q3h.val[0]));- q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[1], m3b)), vreinterpretq_s8_u8(q3h.val[1]));- q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));- q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));-- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];-- scale += 4;-- q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);- q3h.val[1] = vbicq_u8(m2, qhbits.val[1]);- q3h.val[2] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[0]), 1);- q3h.val[3] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[1]), 1);-- q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 4), m3b)), vreinterpretq_s8_u8(q3h.val[0]));- q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 4), m3b)), vreinterpretq_s8_u8(q3h.val[1]));- q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));- q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));-- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];-- scale += 4;-- if (j == 0) {- qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);- qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);- }-- }- sum += d * isum;-- }-- *s = sum;--#elif defined __AVX2__-- const __m256i m3 = _mm256_set1_epi8(3);- const __m256i mone = _mm256_set1_epi8(1);- const __m128i m32 = _mm_set1_epi8(32);-- __m256 acc = _mm256_setzero_ps();-- uint32_t aux[3];-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q3 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- // Set up scales- memcpy(aux, x[i].scales, 12);- __m128i scales128 = _mm_set_epi32(- ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),- ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),- (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),- (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));- scales128 = _mm_sub_epi8(scales128, m32);- const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);- const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);- const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);- const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};-- // high bit- const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);-- // integer accumulator- __m256i sumi = _mm256_setzero_si256();-- int bit = 0;- int is = 0;-- for (int j = 0; j < QK_K/128; ++j) {- // load low 2 bits- const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;-- // prepare low and high bits- const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);- const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);- ++bit;-- const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);- const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);- ++bit;-- const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);- const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);- ++bit;-- const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);- const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);- ++bit;-- // load Q8 quants- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;-- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,- // and 2 if the high bit was set)- __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);- __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);- __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);- __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);-- __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);- __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);- __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);- __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);-- p16_0 = _mm256_sub_epi16(p16_0, q8s_0);- p16_1 = _mm256_sub_epi16(p16_1, q8s_1);- p16_2 = _mm256_sub_epi16(p16_2, q8s_2);- p16_3 = _mm256_sub_epi16(p16_3, q8s_3);-- // multiply with scales- p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);- p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);- p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);- p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);-- // accumulate- p16_0 = _mm256_add_epi32(p16_0, p16_1);- p16_2 = _mm256_add_epi32(p16_2, p16_3);- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));-- }-- // multiply with block scale and accumulate- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);-- }-- *s = hsum_float_8(acc);--#elif defined __AVX__-- const __m128i m3 = _mm_set1_epi8(3);- const __m128i mone = _mm_set1_epi8(1);- const __m128i m32 = _mm_set1_epi8(32);- const __m128i m2 = _mm_set1_epi8(2);-- __m256 acc = _mm256_setzero_ps();-- const uint32_t *aux;-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q3 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- // Set up scales- aux = (const uint32_t *)x[i].scales;- __m128i scales128 = _mm_set_epi32(- ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),- ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),- (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),- (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));- scales128 = _mm_sub_epi8(scales128, m32);- const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);- const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));- const __m128i scales[2] = { scales_0, scales_1 };-- // high bit *128*2 from block_q3_K.hmask[QK_K/8]- const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);- const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);-- // integer accumulator- __m128i sumi_0 = _mm_setzero_si128();- __m128i sumi_1 = _mm_setzero_si128();-- for (int j = 0; j < QK_K/128; ++j) {- // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]- const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;- const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;-- // prepare low and high bits- const int bit = j << 2;-- const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);- const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);- const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);- const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);-- const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);- const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);- const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);- const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);-- const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);- const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);- const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);- const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);-- const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);- const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);- const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);- const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);-- // load Q8 quants from block_q8_K.qs[QK_K]- const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;-- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,- // and 2 if the high bit was set)- __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);- __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);- __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);- __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);- __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);- __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);- __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);- __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);-- __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);- __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);- __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);- __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);- __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);- __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);- __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);- __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);-- p16_0 = _mm_sub_epi16(p16_0, q8s_0);- p16_1 = _mm_sub_epi16(p16_1, q8s_1);- p16_2 = _mm_sub_epi16(p16_2, q8s_2);- p16_3 = _mm_sub_epi16(p16_3, q8s_3);- p16_4 = _mm_sub_epi16(p16_4, q8s_4);- p16_5 = _mm_sub_epi16(p16_5, q8s_5);- p16_6 = _mm_sub_epi16(p16_6, q8s_6);- p16_7 = _mm_sub_epi16(p16_7, q8s_7);-- // multiply with scales- __m128i shuffle = _mm_set1_epi16(0x0100);- p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);- shuffle = _mm_add_epi16(shuffle, m2);- p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);- shuffle = _mm_add_epi16(shuffle, m2);- p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);- shuffle = _mm_add_epi16(shuffle, m2);- p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);- shuffle = _mm_add_epi16(shuffle, m2);- p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);- shuffle = _mm_add_epi16(shuffle, m2);- p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);- shuffle = _mm_add_epi16(shuffle, m2);- p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);- shuffle = _mm_add_epi16(shuffle, m2);- p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);-- // accumulate- p16_0 = _mm_add_epi32(p16_0, p16_1);- p16_2 = _mm_add_epi32(p16_2, p16_3);- p16_4 = _mm_add_epi32(p16_4, p16_5);- p16_6 = _mm_add_epi32(p16_6, p16_7);- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));-- }-- // multiply with block scale and accumulate- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);-- }-- *s = hsum_float_8(acc);--#elif defined __riscv_v_intrinsic-- uint32_t aux[3];- uint32_t utmp[4];-- float sumf = 0;- for (int i = 0; i < nb; ++i) {-- const uint8_t * restrict q3 = x[i].qs;- const uint8_t * restrict qh = x[i].hmask;- const int8_t * restrict q8 = y[i].qs;-- memcpy(aux, x[i].scales, 12);- utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);- utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);- utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);- utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);-- int8_t * scale = (int8_t *)utmp;- for (int j = 0; j < 16; ++j) scale[j] -= 32;--- size_t vl = 32;- uint8_t m = 1;-- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);- vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl);-- int sum_t = 0;-- for (int j = 0; j < QK_K; j += 128) {-- vl = 32;-- // load Q3- vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl);-- vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl));- vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl));- vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl));- vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl));-- // compute mask for subtraction- vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl);- vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl);- vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_m(vmask_0, q3_0, 0x4, vl);- m <<= 1;-- vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);- vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl);- vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_m(vmask_1, q3_1, 0x4, vl);- m <<= 1;-- vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);- vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl);- vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_m(vmask_2, q3_2, 0x4, vl);- m <<= 1;-- vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl);- vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl);- vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_m(vmask_3, q3_3, 0x4, vl);- m <<= 1;-- // load Q8 and take product with Q3- vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl);- vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl);- vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl);- vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl);-- vl = 16;-- // retrieve lane to multiply with scale- vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl);- vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl);- vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl);- vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl);- vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl);- vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl);- vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl);- vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl);-- vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl);- vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl);- vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl);- vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl);-- sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);-- q3 += 32; q8 += 128; scale += 8;-- }-- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;-- sumf += d*sum_t;-- }-- *s = sumf;--#else- // scalar version- // This function is written like this so the compiler can manage to vectorize most of it- // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the- // manually vectorized version above. Every other version I tried would run at least 4 times slower.- // The ideal situation would be if we could just write the code once, and the compiler would- // automatically produce the best possible set of machine instructions, instead of us having to manually- // write vectorized versions for AVX, ARM_NEON, etc.-- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));-- uint32_t auxs[4];- const int8_t * scales = (const int8_t*)auxs;-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q3 = x[i].qs;- const uint8_t * restrict hm = x[i].hmask;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- uint8_t m = 1;- for (int j = 0; j < QK_K; j += 128) {- for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);- a += 32; m <<= 1;- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);- a += 32; m <<= 1;- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);- a += 32; m <<= 1;- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);- a += 32; m <<= 1;- q3 += 32;- }- a = aux8;-- memcpy(auxs, x[i].scales, 12);- uint32_t tmp = auxs[2];- auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);- auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);- auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);- auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);- for (int j = 0; j < QK_K/16; ++j) {- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];- q8 += 8; a += 8;- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;--#endif--}diff --git a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q4_0_q8_0.incdeleted file mode 100644index b99edb44..00000000--- a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc+++ /dev/null@@ -1,366 +0,0 @@-static void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- const int qk = QK8_0;- const int nb = n / qk;-- assert(n % qk == 0);-#if defined(__ARM_FEATURE_MATMUL_INT8)- assert((nrc == 2) || (nrc == 1));-#else- assert(nrc == 1);-#endif- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);-- const block_q4_0 * restrict x = vx;- const block_q8_0 * restrict y = vy;--#if defined(__ARM_FEATURE_MATMUL_INT8)- if (nrc == 2) {- const block_q4_0 * restrict vx0 = vx;- const block_q4_0 * restrict vx1 = vx + bx;-- const block_q8_0 * restrict vy0 = vy;- const block_q8_0 * restrict vy1 = vy + by;-- float32x4_t sumv0 = vdupq_n_f32(0.0f);-- for (int i = 0; i < nb; i++) {- const block_q4_0 * restrict b_x0 = &vx0[i];- const block_q4_0 * restrict b_x1 = &vx1[i];- const block_q8_0 * restrict b_y0 = &vy0[i];- const block_q8_0 * restrict b_y1 = &vy1[i];-- const uint8x16_t m4b = vdupq_n_u8(0x0F);- const int8x16_t s8b = vdupq_n_s8(0x8);-- const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);- const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);-- // 4-bit -> 8-bit- const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));-- // sub 8- const int8x16_t x0_l = vsubq_s8(v0_0l, s8b);- const int8x16_t x0_h = vsubq_s8(v0_0h, s8b);- const int8x16_t x1_l = vsubq_s8(v0_1l, s8b);- const int8x16_t x1_h = vsubq_s8(v0_1h, s8b);-- // load y- const int8x16_t y0_l = vld1q_s8(b_y0->qs);- const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);- const int8x16_t y1_l = vld1q_s8(b_y1->qs);- const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);-- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};-- int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));- int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));-- int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));- int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));-- int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));- int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));-- int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));- int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));-- sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),- l1, r1)), l2, r2)), l3, r3))), scale);- }- float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);- float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);-- vst1_f32(s, vget_low_f32(sumv2));- vst1_f32(s + bs, vget_high_f32(sumv2));- return;- }-#endif-#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q4_0 * restrict x0 = &x[i + 0];- const block_q4_0 * restrict x1 = &x[i + 1];- const block_q8_0 * restrict y0 = &y[i + 0];- const block_q8_0 * restrict y1 = &y[i + 1];-- const uint8x16_t m4b = vdupq_n_u8(0x0F);- const int8x16_t s8b = vdupq_n_s8(0x8);-- const uint8x16_t v0_0 = vld1q_u8(x0->qs);- const uint8x16_t v0_1 = vld1q_u8(x1->qs);-- // 4-bit -> 8-bit- const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));-- // sub 8- const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);- const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);- const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);- const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);-- // load y- const int8x16_t v1_0l = vld1q_s8(y0->qs);- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);- const int8x16_t v1_1l = vld1q_s8(y1->qs);- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);-- // dot product into int32x4_t- const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);- const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));- }-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);-#elif defined(__AVX2__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; ++i) {- /* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );-- __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);-- // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.- const __m256i off = _mm256_set1_epi8( 8 );- qx = _mm256_sub_epi8( qx, off );-- __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_i8_pairs_float_avx2(qx, qy);-- /* Multiply q with scale and accumulate */- acc = _mm256_fmadd_ps( d, q, acc );- }-- *s = hsum_float_8(acc);-#elif defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; ++i) {- // Compute combined scale for the block- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );-- const __m128i lowMask = _mm_set1_epi8(0xF);- const __m128i off = _mm_set1_epi8(8);-- const __m128i tmp = _mm_loadu_si128((const __m128i *)x[i].qs);-- __m128i bx_0 = _mm_and_si128(lowMask, tmp);- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);- bx_0 = _mm_sub_epi8(bx_0, off);- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);-- bx_0 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4));- by_0 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));- bx_0 = _mm_sub_epi8(bx_0, off);- const __m128i i32_1 = mul_sum_i8_pairs(bx_0, by_0);-- // Convert int32_t to float- __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1));-- // Apply the scale, and accumulate- acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);- }-- *s = hsum_float_8(acc);-#elif defined(__SSSE3__)- // set constants- const __m128i lowMask = _mm_set1_epi8(0xF);- const __m128i off = _mm_set1_epi8(8);-- // Initialize accumulator with zeros- __m128 acc_0 = _mm_setzero_ps();- __m128 acc_1 = _mm_setzero_ps();- __m128 acc_2 = _mm_setzero_ps();- __m128 acc_3 = _mm_setzero_ps();-- // First round without accumulation- {- _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 0 and 1- const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) );-- const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[0].qs);-- __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[0].qs);- bx_0 = _mm_sub_epi8(bx_0, off);- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);-- __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));- __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[0].qs + 16));- bx_1 = _mm_sub_epi8(bx_1, off);- const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);-- _mm_prefetch(&x[1] + sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[1] + sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 2 and 3- const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) );-- const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[1].qs);-- __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);- __m128i by_2 = _mm_loadu_si128((const __m128i *)y[1].qs);- bx_2 = _mm_sub_epi8(bx_2, off);- const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);-- __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));- __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[1].qs + 16));- bx_3 = _mm_sub_epi8(bx_3, off);- const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);-- // Convert int32_t to float- __m128 p0 = _mm_cvtepi32_ps(i32_0);- __m128 p1 = _mm_cvtepi32_ps(i32_1);- __m128 p2 = _mm_cvtepi32_ps(i32_2);- __m128 p3 = _mm_cvtepi32_ps(i32_3);-- // Apply the scale- acc_0 = _mm_mul_ps( d_0_1, p0 );- acc_1 = _mm_mul_ps( d_0_1, p1 );- acc_2 = _mm_mul_ps( d_2_3, p2 );- acc_3 = _mm_mul_ps( d_2_3, p3 );- }-- assert(nb % 2 == 0); // TODO: handle odd nb-- // Main loop- for (int i = 2; i < nb; i+=2) {- _mm_prefetch(&x[i] + sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[i] + sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 0 and 1- const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );-- const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[i].qs);-- __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);- bx_0 = _mm_sub_epi8(bx_0, off);- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);-- __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));- __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));- bx_1 = _mm_sub_epi8(bx_1, off);- const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);-- _mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 2 and 3- const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) );-- const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[i + 1].qs);-- __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);- __m128i by_2 = _mm_loadu_si128((const __m128i *)y[i + 1].qs);- bx_2 = _mm_sub_epi8(bx_2, off);- const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);-- __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));- __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[i + 1].qs + 16));- bx_3 = _mm_sub_epi8(bx_3, off);- const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);-- // Convert int32_t to float- __m128 p0 = _mm_cvtepi32_ps(i32_0);- __m128 p1 = _mm_cvtepi32_ps(i32_1);- __m128 p2 = _mm_cvtepi32_ps(i32_2);- __m128 p3 = _mm_cvtepi32_ps(i32_3);-- // Apply the scale- __m128 p0_d = _mm_mul_ps( d_0_1, p0 );- __m128 p1_d = _mm_mul_ps( d_0_1, p1 );- __m128 p2_d = _mm_mul_ps( d_2_3, p2 );- __m128 p3_d = _mm_mul_ps( d_2_3, p3 );-- // Acummulate- acc_0 = _mm_add_ps(p0_d, acc_0);- acc_1 = _mm_add_ps(p1_d, acc_1);- acc_2 = _mm_add_ps(p2_d, acc_2);- acc_3 = _mm_add_ps(p3_d, acc_3);- }-- *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;-- size_t vl = __riscv_vsetvl_e8m1(qk/2);-- for (int i = 0; i < nb; i++) {- // load elements- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);-- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);-- // mask and store lower part of x, and then upper part- vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);- vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);-- vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);-- // subtract offset- vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 8, vl);- vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 8, vl);-- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);-- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);-- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);-- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);-- sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);- }-- *s = sumf;-#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- int sumi = 0;-- for (int j = 0; j < qk/2; ++j) {- const int v0 = (x[i].qs[j] & 0x0F) - 8;- const int v1 = (x[i].qs[j] >> 4) - 8;-- sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);- }-- sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);- }-- *s = sumf;-#endif-}diff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.incdeleted file mode 100644index aeb4c8b9..00000000--- a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc+++ /dev/null@@ -1,360 +0,0 @@-static void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- assert(n % QK_K == 0);- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);-- const block_q4_K * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;-- static const uint32_t kmask1 = 0x3f3f3f3f;- static const uint32_t kmask2 = 0x0f0f0f0f;- static const uint32_t kmask3 = 0x03030303;-- uint32_t utmp[4];--#ifdef __ARM_NEON- const uint8x16_t m4b = vdupq_n_u8(0xf);- const int32x4_t mzero = vdupq_n_s32(0);-- ggml_int8x16x2_t q4bytes;- ggml_int8x16x2_t q8bytes;-- float sumf = 0;-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));-- memcpy(utmp, x[i].scales, 12);-- uint32x2_t mins8 = { 0 };- mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);- mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);-- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[0] &= kmask1;-- const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));- const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),- vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));- sumf -= dmin * vaddvq_s32(prod);-- const uint8_t * scales = (const uint8_t *)utmp;-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- int32_t sumi1 = 0;- int32_t sumi2 = 0;-- for (int j = 0; j < QK_K/64; ++j) {- const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;-- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));-- const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);- sumi1 += vaddvq_s32(p1) * scales[2*j+0];-- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));-- const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);-- sumi2 += vaddvq_s32(p2) * scales[2*j+1];- }-- sumf += d * (sumi1 + sumi2);-- }-- *s = sumf;--#elif defined __AVX2__-- const __m256i m4 = _mm256_set1_epi8(0xF);-- __m256 acc = _mm256_setzero_ps();- __m128 acc_m = _mm_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));-- const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);- const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));- const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);- acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);-- const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);- const __m256i scales = MM256_SET_M128I(sc128, sc128);-- __m256i sumi = _mm256_setzero_si256();-- for (int j = 0; j < QK_K/64; ++j) {-- const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));- const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));-- const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;- const __m256i q4l = _mm256_and_si256(q4bits, m4);- const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);-- const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);- p16l = _mm256_madd_epi16(scale_l, p16l);-- const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);- p16h = _mm256_madd_epi16(scale_h, p16h);- const __m256i sumj = _mm256_add_epi32(p16l, p16h);-- sumi = _mm256_add_epi32(sumi, sumj);- }-- __m256 vd = _mm256_set1_ps(d);- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);-- }-- acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));- acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));-- *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);--#elif defined __AVX__-- const __m128i m4 = _mm_set1_epi8(0xF);- const __m128i m2 = _mm_set1_epi8(0x2);-- __m256 acc = _mm256_setzero_ps();- __m128 acc_m = _mm_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);- const __m128i scales = _mm_cvtepu8_epi16(utmps);- const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));-- const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);- const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);- const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);- const __m128i prod = _mm_madd_epi16(mins, q8s);- acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);-- __m128i sumi_0 = _mm_setzero_si128();- __m128i sumi_1 = _mm_setzero_si128();-- __m128i shuffle = _mm_set1_epi16(0x0100);- for (int j = 0; j < QK_K/64; ++j) {-- const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi16(shuffle, m2);- const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi16(shuffle, m2);-- __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4l_0 = _mm_and_si128(q4bits, m4);- const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);- q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4l_1 = _mm_and_si128(q4bits, m4);- const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);-- const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);- p16l = _mm_madd_epi16(scale_l, p16l);- sumi_0 = _mm_add_epi32(sumi_0, p16l);- const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- p16l = _mm_maddubs_epi16(q4l_1, q8l_1);- p16l = _mm_madd_epi16(scale_l, p16l);- sumi_1 = _mm_add_epi32(sumi_1, p16l);-- const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);- p16h = _mm_madd_epi16(scale_h, p16h);- sumi_0 = _mm_add_epi32(sumi_0, p16h);- const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- p16h = _mm_maddubs_epi16(q4h_1, q8h_1);- p16h = _mm_madd_epi16(scale_h, p16h);- sumi_1 = _mm_add_epi32(sumi_1, p16h);-- }-- __m256 vd = _mm256_set1_ps(d);- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);-- }-- acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));- acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));-- *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);--#elif defined __riscv_v_intrinsic-- const uint8_t * scales = (const uint8_t*)&utmp[0];- const uint8_t * mins = (const uint8_t*)&utmp[2];-- float sumf = 0;-- for (int i = 0; i < nb; ++i) {-- size_t vl = 8;-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);- vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);- vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);- vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));- vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);-- vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);- sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- vl = 32;-- int32_t sum_1 = 0;- int32_t sum_2 = 0;-- vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);-- for (int j = 0; j < QK_K/64; ++j) {- // load Q4- vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);-- // load Q8 and multiply it with lower Q4 nibble- vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);- vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));- vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);- vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);-- sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];-- // load Q8 and multiply it with upper Q4 nibble- vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);- vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));- vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);- vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);-- sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];-- q4 += 32; q8 += 64;-- }-- sumf += d*(sum_1 + sum_2);-- }-- *s = sumf;--#else--- const uint8_t * scales = (const uint8_t*)&utmp[0];- const uint8_t * mins = (const uint8_t*)&utmp[2];-- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- for (int j = 0; j < QK_K/64; ++j) {- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);- a += 32;- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);- a += 32; q4 += 32;- }- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- int sumi = 0;- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];- a = aux8;- int is = 0;- for (int j = 0; j < QK_K/32; ++j) {- int32_t scale = scales[is++];- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;- sumf -= dmin * sumi;- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;-#endif-}diff --git a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q5_0_q8_0.incdeleted file mode 100644index 8ced89f2..00000000--- a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc+++ /dev/null@@ -1,290 +0,0 @@-static void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- const int qk = QK8_0;- const int nb = n / qk;-- assert(n % qk == 0);- assert(qk == QK5_0);- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);-- const block_q5_0 * restrict x = vx;- const block_q8_0 * restrict y = vy;--#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- uint32_t qh0;- uint32_t qh1;-- uint64_t tmp0[4];- uint64_t tmp1[4];-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q5_0 * restrict x0 = &x[i];- const block_q5_0 * restrict x1 = &x[i + 1];- const block_q8_0 * restrict y0 = &y[i];- const block_q8_0 * restrict y1 = &y[i + 1];-- const uint8x16_t m4b = vdupq_n_u8(0x0F);-- // extract the 5th bit via lookup table ((!b) << 4)- memcpy(&qh0, x0->qh, sizeof(qh0));- memcpy(&qh1, x1->qh, sizeof(qh1));-- tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];- tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];- tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];- tmp0[3] = table_b2b_1[(qh0 >> 24) ];-- tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];- tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];- tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];- tmp1[3] = table_b2b_1[(qh1 >> 24) ];-- const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));- const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));- const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));- const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));-- const uint8x16_t v0_0 = vld1q_u8(x0->qs);- const uint8x16_t v0_1 = vld1q_u8(x1->qs);-- // 4-bit -> 8-bit- int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));-- // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)- const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);- const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);- const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);- const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);-- // load y- const int8x16_t v1_0l = vld1q_s8(y0->qs);- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);- const int8x16_t v1_1l = vld1q_s8(y1->qs);- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),- ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),- ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));- }-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);-#elif defined(__wasm_simd128__)- v128_t sumv = wasm_f32x4_splat(0.0f);-- uint32_t qh;- uint64_t tmp[4];-- // TODO: check if unrolling this is better- for (int i = 0; i < nb; ++i) {- const block_q5_0 * restrict x0 = &x[i];- const block_q8_0 * restrict y0 = &y[i];-- const v128_t m4b = wasm_i8x16_splat(0x0F);-- // extract the 5th bit- memcpy(&qh, x0->qh, sizeof(qh));-- tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];- tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];- tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];- tmp[3] = table_b2b_1[(qh >> 24) ];-- const v128_t qhl = wasm_v128_load(tmp + 0);- const v128_t qhh = wasm_v128_load(tmp + 2);-- const v128_t v0 = wasm_v128_load(x0->qs);-- // 4-bit -> 8-bit- const v128_t v0l = wasm_v128_and (v0, m4b);- const v128_t v0h = wasm_u8x16_shr(v0, 4);-- // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)- const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);- const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);-- // load y- const v128_t v1l = wasm_v128_load(y0->qs);- const v128_t v1h = wasm_v128_load(y0->qs + 16);-- // int8x16 -> int16x8- const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);- const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);- const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);- const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);-- const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);- const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);- const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);- const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);-- // dot product- sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(- wasm_i32x4_add(- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),- wasm_i32x4_dot_i16x8(v0lfh, v1lh)),- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),- wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),- wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));- }-- *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +- wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);-#elif defined(__AVX2__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; i++) {- /* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));-- __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);- __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);- bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));- qx = _mm256_or_si256(qx, bxhi);-- __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_i8_pairs_float_avx2(qx, qy);-- /* Multiply q with scale and accumulate */- acc = _mm256_fmadd_ps(d, q, acc);- }-- *s = hsum_float_8(acc);-#elif defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();- __m128i mask = _mm_set1_epi8((char)0xF0);-- // Main loop- for (int i = 0; i < nb; i++) {- /* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));-- __m256i bx_0 = bytes_from_nibbles_32_avx(x[i].qs);- const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);- __m128i bxhil = _mm256_castsi256_si128(bxhi);- __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);- bxhil = _mm_andnot_si128(bxhil, mask);- bxhih = _mm_andnot_si128(bxhih, mask);- __m128i bxl = _mm256_castsi256_si128(bx_0);- __m128i bxh = _mm256_extractf128_si256(bx_0, 1);- bxl = _mm_or_si128(bxl, bxhil);- bxh = _mm_or_si128(bxh, bxhih);- bx_0 = MM256_SET_M128I(bxh, bxl);-- const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_i8_pairs_float_avx(bx_0, by_0);-- /* Multiply q with scale and accumulate */- acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);- }-- *s = hsum_float_8(acc);-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;-- uint32_t qh;-- size_t vl = __riscv_vsetvl_e8m1(qk/2);-- // These temporary registers are for masking and shift operations- vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);- vuint32m2_t vt_2 = __riscv_vsll_vv_u32m2(__riscv_vmv_v_x_u32m2(1, vl), vt_1, vl);-- vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);- vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);-- for (int i = 0; i < nb; i++) {- memcpy(&qh, x[i].qh, sizeof(uint32_t));-- // ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;- vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);- vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(xha_0, vt_1, vl);- vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);-- // ((qh & (1u << (j + 16))) >> (j + 12));- vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(vt_3, qh, vl);- vuint32m2_t xhl_1 = __riscv_vsrl_vv_u32m2(xha_1, vt_4, vl);-- // narrowing- vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xhl_0, vl);- vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);-- vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xhl_1, vl);- vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);-- // load- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);-- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);-- vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);- vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);-- vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);- vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);-- vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);-- vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 16, vl);- vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 16, vl);-- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);-- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);-- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);-- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);-- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;- }-- *s = sumf;-#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- uint32_t qh;- memcpy(&qh, x[i].qh, sizeof(qh));-- int sumi = 0;-- for (int j = 0; j < qk/2; ++j) {- const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;- const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));-- const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;- const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;-- sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);- }-- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;- }-- *s = sumf;-#endif-}diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.incdeleted file mode 100644index 2393e0d2..00000000--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc+++ /dev/null@@ -1,422 +0,0 @@-static void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- assert(n % QK_K == 0);- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);-- const block_q5_K * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;-- static const uint32_t kmask1 = 0x3f3f3f3f;- static const uint32_t kmask2 = 0x0f0f0f0f;- static const uint32_t kmask3 = 0x03030303;-- uint32_t utmp[4];--#ifdef __ARM_NEON- const uint8x16_t m4b = vdupq_n_u8(0xf);- const uint8x16_t mone = vdupq_n_u8(1);- const uint8x16_t mtwo = vdupq_n_u8(2);- const int32x4_t mzero = vdupq_n_s32(0);-- ggml_int8x16x4_t q5bytes;-- float sumf = 0;-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- const uint8x8_t mins8 = vld1_u8((const uint8_t*)utmp + 8);- const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(mins8));- const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),- vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));- int32_t sumi_mins = vaddvq_s32(prod);-- const uint8_t * scales = (const uint8_t *)utmp;-- const uint8_t * restrict q5 = x[i].qs;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);-- ggml_uint8x16x4_t q5h;-- int32_t sumi = 0;-- for (int j = 0; j < QK_K/64; ++j) {-- const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;-- q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);- q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);- q5h.val[2] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[0]), 3);- q5h.val[3] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[1]), 3);- qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 2);- qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 2);-- q5bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[0], m4b), q5h.val[0]));- q5bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[1], m4b), q5h.val[1]));- q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));- q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));-- sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;- sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;- }-- sumf += d * sumi - dmin * sumi_mins;- }-- *s = sumf;--#elif defined __AVX2__-- const __m256i m4 = _mm256_set1_epi8(0xF);- const __m128i mzero = _mm_setzero_si128();- const __m256i mone = _mm256_set1_epi8(1);-- __m256 acc = _mm256_setzero_ps();-- float summs = 0.f;-- for (int i = 0; i < nb; ++i) {-- const uint8_t * restrict q5 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;--#if QK_K == 256- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-#else- // TODO- const float d = 0, dmin = 0;-#endif-- const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));-- const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);- const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));- const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);- const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);- summs += dmin * _mm_extract_epi32(hsum, 0);-- const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);- const __m256i scales = MM256_SET_M128I(sc128, sc128);-- const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);- __m256i hmask = mone;-- __m256i sumi = _mm256_setzero_si256();-- int bit = 0;-- for (int j = 0; j < QK_K/64; ++j) {-- const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));- const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));-- const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;-- const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);- const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);- const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);- hmask = _mm256_slli_epi16(hmask, 1);-- const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);- const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);- const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);- hmask = _mm256_slli_epi16(hmask, 1);-- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;-- __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);- __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);-- p16_0 = _mm256_madd_epi16(scale_0, p16_0);- p16_1 = _mm256_madd_epi16(scale_1, p16_1);-- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));-- }-- __m256 vd = _mm256_set1_ps(d);- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);-- }-- *s = hsum_float_8(acc) + summs;--#elif defined __AVX__-- const __m128i m4 = _mm_set1_epi8(0xF);- const __m128i mzero = _mm_setzero_si128();- const __m128i mone = _mm_set1_epi8(1);- const __m128i m2 = _mm_set1_epi8(2);-- __m256 acc = _mm256_setzero_ps();-- float summs = 0.f;-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- const uint8_t * restrict q5 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);- const __m128i scales = _mm_cvtepu8_epi16(utmps);- const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));-- const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);- const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);- const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);- const __m128i prod = _mm_madd_epi16(mins, q8s);- const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);- summs += dmin * _mm_extract_epi32(hsum, 0);-- const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);- const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);- __m128i hmask = mone;-- __m128i sumi_0 = _mm_setzero_si128();- __m128i sumi_1 = _mm_setzero_si128();-- int bit = 0;-- __m128i shuffle = _mm_set1_epi16(0x0100);- for (int j = 0; j < QK_K/64; ++j) {-- const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi16(shuffle, m2);- const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi16(shuffle, m2);-- const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;- const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;-- __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);- __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);- __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);- __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);- __m128i q5_0 = _mm_add_epi8(q5l_0, q5h_0);- __m128i q5_1 = _mm_add_epi8(q5l_1, q5h_1);- hmask = _mm_slli_epi16(hmask, 1);-- __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);- __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);- p16_0 = _mm_madd_epi16(scale_0, p16_0);- p16_1 = _mm_madd_epi16(scale_0, p16_1);-- q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);- q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);- q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);- q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);- q5_0 = _mm_add_epi8(q5l_0, q5h_0);- q5_1 = _mm_add_epi8(q5l_1, q5h_1);- hmask = _mm_slli_epi16(hmask, 1);-- q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);- __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);- p16_2 = _mm_madd_epi16(scale_1, p16_2);- p16_3 = _mm_madd_epi16(scale_1, p16_3);-- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));-- }-- __m256 vd = _mm256_set1_ps(d);- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);-- }-- *s = hsum_float_8(acc) + summs;--#elif defined __riscv_v_intrinsic-- const uint8_t * scales = (const uint8_t*)&utmp[0];- const uint8_t * mins = (const uint8_t*)&utmp[2];-- float sumf = 0;- float sums = 0.0;-- size_t vl;-- for (int i = 0; i < nb; ++i) {-- vl = 8;-- const uint8_t * restrict q5 = x[i].qs;- const uint8_t * restrict hm = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;-- vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);- vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);- vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);- vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));- vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);-- vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);- sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);-- vl = 32;- int32_t aux32 = 0;- int is = 0;-- uint8_t m = 1;- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);- vuint8m1_t vqh = __riscv_vle8_v_u8m1(hm, vl);-- for (int j = 0; j < QK_K/64; ++j) {- // load Q5 and Q8- vuint8m1_t q5_x = __riscv_vle8_v_u8m1(q5, vl);- vint8m1_t q8_y1 = __riscv_vle8_v_i8m1(q8, vl);- vint8m1_t q8_y2 = __riscv_vle8_v_i8m1(q8+32, vl);-- // compute mask for addition- vint8m1_t q5_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q5_x, 0x0F, vl));- vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);- vbool8_t vmask_1 = __riscv_vmsne_vx_u8m1_b8(qh_m1, 0, vl);- vint8m1_t q5_m1 = __riscv_vadd_vx_i8m1_m(vmask_1, q5_a, 16, vl);- m <<= 1;-- vint8m1_t q5_l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q5_x, 0x04, vl));- vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);- vbool8_t vmask_2 = __riscv_vmsne_vx_u8m1_b8(qh_m2, 0, vl);- vint8m1_t q5_m2 = __riscv_vadd_vx_i8m1_m(vmask_2, q5_l, 16, vl);- m <<= 1;-- vint16m2_t v0 = __riscv_vwmul_vv_i16m2(q5_m1, q8_y1, vl);- vint16m2_t v1 = __riscv_vwmul_vv_i16m2(q5_m2, q8_y2, vl);-- vint32m4_t vs1 = __riscv_vwmul_vx_i32m4(v0, scales[is++], vl);- vint32m4_t vs2 = __riscv_vwmul_vx_i32m4(v1, scales[is++], vl);-- vint32m1_t vacc1 = __riscv_vredsum_vs_i32m4_i32m1(vs1, vzero, vl);- vint32m1_t vacc2 = __riscv_vredsum_vs_i32m4_i32m1(vs2, vzero, vl);-- aux32 += __riscv_vmv_x_s_i32m1_i32(vacc1) + __riscv_vmv_x_s_i32m1_i32(vacc2);- q5 += 32; q8 += 64;-- }-- vfloat32m1_t vaux = __riscv_vfmul_vf_f32m1(__riscv_vfmv_v_f_f32m1(aux32, 1), d, 1);- sums += __riscv_vfmv_f_s_f32m1_f32(vaux);-- }-- *s = sumf+sums;--#else-- const uint8_t * scales = (const uint8_t*)&utmp[0];- const uint8_t * mins = (const uint8_t*)&utmp[2];-- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].qs;- const uint8_t * restrict hm = x[i].qh;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- uint8_t m = 1;- for (int j = 0; j < QK_K/64; ++j) {- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);- a += 32; m <<= 1;- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);- a += 32; m <<= 1;- q4 += 32;- }- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- int sumi = 0;- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];- a = aux8;- int is = 0;- for (int j = 0; j < QK_K/32; ++j) {- int32_t scale = scales[is++];- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;- sumf -= dmin * sumi;- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;-#endif-}diff --git a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.incdeleted file mode 100644index 343d633d..00000000--- a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc+++ /dev/null@@ -1,434 +0,0 @@-static void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- assert(n % QK_K == 0);- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);-- const block_q6_K * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;--#ifdef __ARM_NEON- float sum = 0;-- const uint8x16_t m4b = vdupq_n_u8(0xF);- const int32x4_t vzero = vdupq_n_s32(0);- //const int8x16_t m32s = vdupq_n_s8(32);-- const uint8x16_t mone = vdupq_n_u8(3);-- ggml_int8x16x4_t q6bytes;- ggml_uint8x16x4_t q6h;-- for (int i = 0; i < nb; ++i) {-- const float d_all = GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q6 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const int8_t * restrict scale = x[i].scales;-- const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);- const int8x16_t scales = vld1q_s8(scale);- const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};-- const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),- vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),- vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),- vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));- int32_t isum_mins = vaddvq_s32(prod);-- int32_t isum = 0;-- for (int j = 0; j < QK_K/128; ++j) {-- ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;- ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;- ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;-- q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);- q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);- uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);- q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[1], 2);- q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);-- //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);- //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);- //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);- //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);- q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));- q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));- q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));- q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));-- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];-- scale += 4;-- q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;-- shifted = vshrq_n_u8(qhbits.val[0], 4);- q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[1], 4);- q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[0], 6);- q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[1], 6);- q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);-- //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);- //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);- //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);- //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);- q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));- q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));- q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));- q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));-- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];- scale += 4;- }- //sum += isum * d_all * y[i].d;- sum += d_all * y[i].d * (isum - 32 * isum_mins);-- }- *s = sum;--#elif defined __AVX2__-- const __m256i m4 = _mm256_set1_epi8(0xF);- const __m256i m2 = _mm256_set1_epi8(3);- const __m256i m32s = _mm256_set1_epi8(32);-- __m256 acc = _mm256_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q4 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);-- __m256i sumi = _mm256_setzero_si256();-- int is = 0;-- for (int j = 0; j < QK_K/128; ++j) {-- const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));- const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));- const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));- const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));- is += 4;-- const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;- const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;- const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;-- const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);- const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);- const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);- const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);-- const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);- const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);- const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);- const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);-- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;-- __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);- __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);- __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);- __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);-- __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);- __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);- __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);- __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);-- p16_0 = _mm256_sub_epi16(p16_0, q8s_0);- p16_1 = _mm256_sub_epi16(p16_1, q8s_1);- p16_2 = _mm256_sub_epi16(p16_2, q8s_2);- p16_3 = _mm256_sub_epi16(p16_3, q8s_3);-- p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);- p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);- p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);- p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);-- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));-- }-- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);- }-- *s = hsum_float_8(acc);--#elif defined __AVX__-- const __m128i m4 = _mm_set1_epi8(0xF);- const __m128i m3 = _mm_set1_epi8(3);- const __m128i m32s = _mm_set1_epi8(32);- const __m128i m2 = _mm_set1_epi8(2);-- __m256 acc = _mm256_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q4 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);-- __m128i sumi_0 = _mm_setzero_si128();- __m128i sumi_1 = _mm_setzero_si128();-- __m128i shuffle = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);- for (int j = 0; j < QK_K/128; ++j) {-- const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;- const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;-- const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);- const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);- const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 2), m3), 4);- const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 2), m3), 4);- const __m128i q4h_4 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 4), m3), 4);- const __m128i q4h_5 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 4), m3), 4);- const __m128i q4h_6 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 6), m3), 4);- const __m128i q4h_7 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 6), m3), 4);-- const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;-- const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m4), q4h_0);- const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m4), q4h_1);- const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m4), q4h_2);- const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m4), q4h_3);- const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m4), q4h_4);- const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m4), q4h_5);- const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m4), q4h_6);- const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m4), q4h_7);-- const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;-- __m128i q8s_0 = _mm_maddubs_epi16(m32s, q8_0);- __m128i q8s_1 = _mm_maddubs_epi16(m32s, q8_1);- __m128i q8s_2 = _mm_maddubs_epi16(m32s, q8_2);- __m128i q8s_3 = _mm_maddubs_epi16(m32s, q8_3);- __m128i q8s_4 = _mm_maddubs_epi16(m32s, q8_4);- __m128i q8s_5 = _mm_maddubs_epi16(m32s, q8_5);- __m128i q8s_6 = _mm_maddubs_epi16(m32s, q8_6);- __m128i q8s_7 = _mm_maddubs_epi16(m32s, q8_7);-- __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);- __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);- __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);- __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);- __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);- __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);- __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);- __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);-- p16_0 = _mm_sub_epi16(p16_0, q8s_0);- p16_1 = _mm_sub_epi16(p16_1, q8s_1);- p16_2 = _mm_sub_epi16(p16_2, q8s_2);- p16_3 = _mm_sub_epi16(p16_3, q8s_3);- p16_4 = _mm_sub_epi16(p16_4, q8s_4);- p16_5 = _mm_sub_epi16(p16_5, q8s_5);- p16_6 = _mm_sub_epi16(p16_6, q8s_6);- p16_7 = _mm_sub_epi16(p16_7, q8s_7);-- const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);- const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);- const __m128i scale_2 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);- const __m128i scale_3 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);-- p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);- p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1);- p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);- p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3);- p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);- p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_2, scale_2)), p16_5);- p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);- p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_3, scale_3)), p16_7);-- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));-- }-- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);- }-- *s = hsum_float_8(acc);--#elif defined __riscv_v_intrinsic-- float sumf = 0;- for (int i = 0; i < nb; ++i) {-- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;-- const uint8_t * restrict q6 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const int8_t * restrict scale = x[i].scales;-- size_t vl;-- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);-- int sum_t = 0;- int is = 0;-- for (int j = 0; j < QK_K/128; ++j) {-- vl = 32;-- // load qh- vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);-- // load Q6- vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);- vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);-- vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);- vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);- vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);- vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);-- vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);- vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);- vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);- vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);-- vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);- vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);- vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);- vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);-- vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);- vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);- vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);- vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);-- // load Q8 and take product- vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);- vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);- vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);- vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);-- vl = 16;-- vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);- vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);- vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);- vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);- vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);- vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);- vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);- vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);-- vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);- vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);- vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);- vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);-- sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);-- q6 += 64; qh += 32; q8 += 128; is=8;-- }-- sumf += d * sum_t;-- }-- *s = sumf;--#else-- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- for (int j = 0; j < QK_K; j += 128) {- for (int l = 0; l < 32; ++l) {- a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;- a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;- a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;- a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;- }- a += 128;- q4 += 64;- qh += 32;- }- a = aux8;- int is = 0;- for (int j = 0; j < QK_K/16; ++j) {- int scale = x[i].scales[is++];- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;-#endif-}diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.incdeleted file mode 100644index 46961faf..00000000--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc+++ /dev/null@@ -1,169 +0,0 @@-static void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- const int qk = QK8_0;- const int nb = n / qk;-- assert(n % qk == 0);-#if defined(__ARM_FEATURE_MATMUL_INT8)- assert((nrc == 2) || (nrc == 1));-#else- assert(nrc == 1);-#endif- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);-- const block_q8_0 * restrict x = vx;- const block_q8_0 * restrict y = vy;--#if defined(__ARM_FEATURE_MATMUL_INT8)- if (nrc == 2) {- const block_q8_0 * restrict vx0 = vx;- const block_q8_0 * restrict vx1 = vx + bx;- const block_q8_0 * restrict vy0 = vy;- const block_q8_0 * restrict vy1 = vy + by;-- float32x4_t sumv0 = vdupq_n_f32(0.0f);-- for (int i = 0; i < nb; i++) {- const block_q8_0 * restrict b_x0 = &vx0[i];- const block_q8_0 * restrict b_y0 = &vy0[i];-- const block_q8_0 * restrict b_x1 = &vx1[i];- const block_q8_0 * restrict b_y1 = &vy1[i];-- const int8x16_t x0_l = vld1q_s8(b_x0->qs);- const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16);- const int8x16_t x1_l = vld1q_s8(b_x1->qs);- const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16);-- // load y- const int8x16_t y0_l = vld1q_s8(b_y0->qs);- const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);- const int8x16_t y1_l = vld1q_s8(b_y1->qs);- const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);-- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};-- int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));- int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));-- int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));- int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));-- int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));- int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));-- int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));- int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));-- sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),- l1, r1)), l2, r2)), l3, r3))), scale);- }- float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);- float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);-- vst1_f32(s, vget_low_f32(sumv2));- vst1_f32(s + bs, vget_high_f32(sumv2));- return;- }-#endif-#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q8_0 * restrict x0 = &x[i + 0];- const block_q8_0 * restrict x1 = &x[i + 1];- const block_q8_0 * restrict y0 = &y[i + 0];- const block_q8_0 * restrict y1 = &y[i + 1];-- const int8x16_t x0_0 = vld1q_s8(x0->qs);- const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);- const int8x16_t x1_0 = vld1q_s8(x1->qs);- const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);-- // load y- const int8x16_t y0_0 = vld1q_s8(y0->qs);- const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);- const int8x16_t y1_0 = vld1q_s8(y1->qs);- const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),- ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));-- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),- ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));- }-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);-#elif defined(__AVX2__) || defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; ++i) {- // Compute combined scale for the block- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));- __m256i qx = _mm256_loadu_si256((const __m256i *)x[i].qs);- __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);--#ifdef __AVX2__- const __m256 q = mul_sum_i8_pairs_float_avx2(qx, qy);-#else- const __m256 q = mul_sum_i8_pairs_float_avx(qx, qy);-#endif-- // Multiply q with scale and accumulate-#if defined(__AVX2__)- acc = _mm256_fmadd_ps( d, q, acc );-#else- acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );-#endif- }-- *s = hsum_float_8(acc);-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;- size_t vl = __riscv_vsetvl_e8m1(qk);-- for (int i = 0; i < nb; i++) {- // load elements- vint8m1_t bx_0 = __riscv_vle8_v_i8m1(x[i].qs, vl);- vint8m1_t by_0 = __riscv_vle8_v_i8m1(y[i].qs, vl);-- vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx_0, by_0, vl);-- vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);- vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl);-- int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);-- sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));- }-- *s = sumf;-#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- int sumi = 0;-- for (int j = 0; j < qk; j++) {- sumi += x[i].qs[j]*y[i].qs[j];- }-- sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));- }-- *s = sumf;-#endif-}diff --git a/llama.cpp/imatrix/imatrix.cpp b/llama.cpp/imatrix/imatrix.cppindex 74576f01..18dd537e 100644--- a/llama.cpp/imatrix/imatrix.cpp+++ b/llama.cpp/imatrix/imatrix.cpp@@ -27,6 +27,7 @@ struct Stats {};struct StatParams {+ std::string dataset;std::string ofile = "imatrix.dat";int n_output_frequency = 10;int verbosity = 1;@@ -50,7 +51,7 @@ private:std::vector<float> m_src1_data;std::vector<char> m_ids; // the expert ids from ggml_mul_mat_id//- void save_imatrix(const char * file_name) const;+ void save_imatrix(const char * file_name, const char * dataset) const;void keep_imatrix(int ncall) const;};@@ -203,7 +204,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void *}void IMatrixCollector::save_imatrix() const {- save_imatrix(m_params.ofile.empty() ? "imatrix.dat" : m_params.ofile.c_str());+ save_imatrix(m_params.ofile.empty() ? "imatrix.dat" : m_params.ofile.c_str(), m_params.dataset.c_str());}void IMatrixCollector::keep_imatrix(int ncall) const {@@ -211,24 +212,33 @@ void IMatrixCollector::keep_imatrix(int ncall) const {if (file_name.empty()) file_name = "imatrix.dat";file_name += ".at_";file_name += std::to_string(ncall);- save_imatrix(file_name.c_str());+ save_imatrix(file_name.c_str(), m_params.dataset.c_str());}-void IMatrixCollector::save_imatrix(const char * fname) const {+void IMatrixCollector::save_imatrix(const char * fname, const char * dataset) const {std::ofstream out(fname, std::ios::binary);int n_entries = m_stats.size();- out.write((const char*)&n_entries, sizeof(n_entries));- for (auto& p : m_stats) {+ out.write((const char *) &n_entries, sizeof(n_entries));+ for (const auto & p : m_stats) {int len = p.first.size();- out.write((const char*)&len, sizeof(len));+ out.write((const char *) &len, sizeof(len));out.write(p.first.c_str(), len);- out.write((const char*)&p.second.ncall, sizeof(p.second.ncall));+ out.write((const char *) &p.second.ncall, sizeof(p.second.ncall));int nval = p.second.values.size();- out.write((const char*)&nval, sizeof(nval));- if (nval > 0) out.write((const char*)p.second.values.data(), nval*sizeof(float));+ out.write((const char *) &nval, sizeof(nval));+ if (nval > 0) out.write((const char *) p.second.values.data(), nval * sizeof(float));}++ // Write the number of call the matrix was computed with+ out.write((const char *) &m_last_call, sizeof(m_last_call));++ // Write the dataset name at the end of the file to later on specify it in quantize+ int n_dataset = strlen(dataset);+ out.write((const char *) &n_dataset, sizeof(n_dataset));+ out.write(dataset, n_dataset);+if (m_params.verbosity > 0) {- fprintf(stderr, "\n%s: stored collected data after %d chunks in %s\n",__func__,m_last_call,fname);+ fprintf(stderr, "\n%s: stored collected data after %d chunks in %s\n", __func__, m_last_call, fname);}}@@ -565,6 +575,29 @@ int main(int argc, char ** argv) {}}+ gpt_params params;+ params.n_batch = 512;+ if (!gpt_params_parse(args.size(), args.data(), params)) {+ return 1;+ }++ params.logits_all = true;+ params.n_batch = std::min(params.n_batch, params.n_ctx);++ print_build_info();++ if (params.seed == LLAMA_DEFAULT_SEED) {+ params.seed = time(NULL);+ }++ fprintf(stderr, "%s: seed = %u\n", __func__, params.seed);++ std::mt19937 rng(params.seed);+ if (params.random_prompt) {+ params.prompt = gpt_random_prompt(rng);+ }++ sparams.dataset = params.prompt_file;g_collector.set_parameters(std::move(sparams));if (!combine_files.empty()) {@@ -603,28 +636,6 @@ int main(int argc, char ** argv) {}}- gpt_params params;- params.n_batch = 512;- if (!gpt_params_parse(args.size(), args.data(), params)) {- return 1;- }-- params.logits_all = true;- params.n_batch = std::min(params.n_batch, params.n_ctx);-- print_build_info();-- if (params.seed == LLAMA_DEFAULT_SEED) {- params.seed = time(NULL);- }-- fprintf(stderr, "%s: seed = %u\n", __func__, params.seed);-- std::mt19937 rng(params.seed);- if (params.random_prompt) {- params.prompt = gpt_random_prompt(rng);- }-llama_backend_init();llama_numa_init(params.numa);diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex c788c059..58ed01a4 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -21,6 +21,7 @@#endif#include <algorithm>+#include <future>#include <array>#include <cassert>#include <cinttypes>@@ -268,6 +269,7 @@ enum llm_kv {LLM_KV_SSM_TIME_STEP_RANK,LLM_KV_TOKENIZER_MODEL,+ LLM_KV_TOKENIZER_PRE,LLM_KV_TOKENIZER_LIST,LLM_KV_TOKENIZER_TOKEN_TYPE,LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,@@ -344,6 +346,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },+ { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },{ LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },@@ -1645,7 +1648,7 @@ struct llama_hparams {float f_logit_scale = 0.0f;bool causal_attn = true;- bool need_kq_pos = false;+ bool use_alibi = false; // currently, we need KQ_pos data for ALiBi-based modelsenum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE;enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE;@@ -1735,6 +1738,7 @@ struct llama_cparams {bool embeddings;bool causal_attn;bool offload_kqv;+ bool flash_attn;enum llama_pooling_type pooling_type;@@ -1838,8 +1842,8 @@ struct llama_kv_cache {bool has_shift = false;bool do_defrag = false;bool do_copy = false;- // with recurrent state models, a cell can hold the state for more than one past token- bool recurrent = false;+ bool recurrent = false; // with recurrent state models, a cell can hold the state for more than one past token+ bool v_trans = true; // the value tensor is transposed// Note: The value of head isn't only used to optimize searching// for a free KV slot. llama_decode_internal also uses it, so it@@ -1916,7 +1920,8 @@ struct llama_vocab {ttype type;};- enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;+ enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;+ enum llama_vocab_pre_type type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;std::unordered_map<token, id> token_to_id;std::vector<token_data> id_to_token;@@ -2137,11 +2142,14 @@ struct llama_context {static bool llama_kv_cache_init(struct llama_kv_cache & cache,- const llama_model & model,+ const llama_context * ctx,ggml_type type_k,ggml_type type_v,uint32_t kv_size,bool offload) {+ const llama_model & model = ctx->model;+ const llama_cparams & cparams = ctx->cparams;+const struct llama_hparams & hparams = model.hparams;const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s();@@ -2152,6 +2160,7 @@ static bool llama_kv_cache_init(// TODO: find a nicer way to add other recurrent model architecturescache.recurrent = model.arch == LLM_ARCH_MAMBA;+ cache.v_trans = !cparams.flash_attn;// TODO: support mixed reccurent Transformer architectues// NOTE: (!a || b) is a logical implication (a -> b)@@ -2364,6 +2373,10 @@ static void llama_kv_cache_clear(struct llama_kv_cache & cache) {}cache.head = 0;cache.used = 0;++ for (auto & buf : cache.bufs) {+ ggml_backend_buffer_clear(buf, 0);+ }}static bool llama_kv_cache_seq_rm(@@ -2684,6 +2697,7 @@ namespace GGUFMeta {case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";+ case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";}return "unknown";}@@ -2695,13 +2709,16 @@ namespace GGUFMeta {__func__, override_type_to_str(ovrd->tag), ovrd->key);switch (ovrd->tag) {case LLAMA_KV_OVERRIDE_TYPE_BOOL: {- LLAMA_LOG_INFO("%s\n", ovrd->bool_value ? "true" : "false");+ LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");} break;case LLAMA_KV_OVERRIDE_TYPE_INT: {- LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->int_value);+ LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);} break;case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {- LLAMA_LOG_INFO("%.6f\n", ovrd->float_value);+ LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);+ } break;+ case LLAMA_KV_OVERRIDE_TYPE_STR: {+ LLAMA_LOG_INFO("%s\n", ovrd->val_str);} break;default:// Shouldn't be possible to end up here, but just in case...@@ -2720,7 +2737,7 @@ namespace GGUFMeta {static typename std::enable_if<std::is_same<OT, bool>::value, bool>::typetry_override(OT & target, const struct llama_model_kv_override * ovrd) {if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {- target = ovrd->bool_value;+ target = ovrd->val_bool;return true;}return false;@@ -2730,7 +2747,7 @@ namespace GGUFMeta {static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::typetry_override(OT & target, const struct llama_model_kv_override * ovrd) {if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {- target = ovrd->int_value;+ target = ovrd->val_i64;return true;}return false;@@ -2740,7 +2757,7 @@ namespace GGUFMeta {static typename std::enable_if<std::is_floating_point<OT>::value, bool>::typetry_override(T & target, const struct llama_model_kv_override * ovrd) {if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {- target = ovrd->float_value;+ target = ovrd->val_f64;return true;}return false;@@ -2749,12 +2766,11 @@ namespace GGUFMeta {template<typename OT>static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::typetry_override(T & target, const struct llama_model_kv_override * ovrd) {- (void)target;- (void)ovrd;- if (!ovrd) { return false; }- // Currently, we should never end up here so it would be a bug if we do.- throw std::runtime_error(format("Unsupported attempt to override string type for metadata key %s\n",- ovrd ? ovrd->key : "NULL"));+ if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {+ target = ovrd->val_str;+ return true;+ }+ return false;}static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {@@ -2787,6 +2803,7 @@ struct llama_model_loader {size_t n_bytes = 0;bool use_mmap = false;+ bool check_tensors;llama_files files;llama_ftype ftype;@@ -2820,7 +2837,7 @@ struct llama_model_loader {std::string arch_name;LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);- llama_model_loader(const std::string & fname, bool use_mmap, const struct llama_model_kv_override * param_overrides_p) {+ llama_model_loader(const std::string & fname, bool use_mmap, bool check_tensors, const struct llama_model_kv_override * param_overrides_p) {int trace = 0;if (getenv("LLAMA_TRACE")) {trace = atoi(getenv("LLAMA_TRACE"));@@ -2919,9 +2936,17 @@ struct llama_model_loader {fver = (enum llama_fver) gguf_get_version(meta);+ std::set<std::string> tensor_names;for (auto & w : weights) {n_elements += ggml_nelements(w.tensor);n_bytes += ggml_nbytes(w.tensor);+ // make sure there is no duplicated tensor names+ const std::string name(w.tensor->name);+ auto found = tensor_names.find(name);+ if (found != tensor_names.end()) {+ throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", w.tensor->name));+ }+ tensor_names.insert(name);}LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",@@ -3028,6 +3053,7 @@ struct llama_model_loader {}this->use_mmap = use_mmap;+ this->check_tensors = check_tensors;}~llama_model_loader() {@@ -3286,6 +3312,10 @@ struct llama_model_loader {file->seek(w.offs, SEEK_SET);file->read_raw(cur->data, ggml_nbytes(cur));}++ if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {+ throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));+ }}size_t size_done = 0;@@ -3302,6 +3332,8 @@ struct llama_model_loader {GGML_ASSERT(size_data != 0 && "call init_mappings() first");std::vector<no_init<uint8_t>> read_buf;+ std::vector<std::future<std::pair<ggml_tensor *, bool>>> validation_result;+for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {const auto * weight = get_weight(ggml_get_name(cur));if (weight == nullptr) {@@ -3323,37 +3355,66 @@ struct llama_model_loader {if (bufs_mmap.count(weight->idx)) {buf_mmap = bufs_mmap.at(weight->idx);}+ uint8_t * data = (uint8_t *) mapping->addr + weight->offs;++ if (check_tensors) {+ validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {+ return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size));+ }));+ }+GGML_ASSERT(buf_mmap || cur->data); // either we have a buffer to allocate the tensor in, or it is already allocatedif (buf_mmap && cur->data == nullptr) {- ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + weight->offs);+ ggml_backend_tensor_alloc(buf_mmap, cur, data);if (lmlocks) {const auto & lmlock = lmlocks->at(weight->idx);- lmlock->grow_to(weight->offs + ggml_nbytes(cur));+ lmlock->grow_to(weight->offs + n_size);}auto & mmap_used = mmaps_used[weight->idx];mmap_used.first = std::min(mmap_used.first, weight->offs);mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);} else {- ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + weight->offs, 0, n_size);+ ggml_backend_tensor_set(cur, data, 0, n_size);}} else {GGML_ASSERT(weight->idx < files.size());const auto & file = files.at(weight->idx);if (ggml_backend_buffer_is_host(cur->buffer)) {file->seek(weight->offs, SEEK_SET);- file->read_raw(cur->data, ggml_nbytes(cur));+ file->read_raw(cur->data, n_size);+ if (check_tensors) {+ validation_result.emplace_back(std::async(std::launch::async, [cur, n_size] {+ return std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size));+ }));+ }} else {- read_buf.resize(ggml_nbytes(cur));+ read_buf.resize(n_size);file->seek(weight->offs, SEEK_SET);- file->read_raw(read_buf.data(), ggml_nbytes(cur));+ file->read_raw(read_buf.data(), n_size);ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);+ if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {+ throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));+ }}}size_done += n_size;}+ // check validation results+ bool validation_failed = false;+ for (auto & future : validation_result) {+ auto result = future.get();+ if (!result.second) {+ LLAMA_LOG_ERROR("%s: tensor '%s' has invalid data\n", __func__, ggml_get_name(result.first));+ validation_failed = true;+ }+ }+ if (validation_failed) {+ throw std::runtime_error("found tensors with invalid data");+ }+// check if this is the last call and do final cleanupif (size_done >= size_data) {// unmap offloaded tensors and metadata@@ -3948,7 +4009,7 @@ static void llm_load_hparams(model.ftype = ml.ftype;if (hparams.f_max_alibi_bias > 0.0f) {- hparams.need_kq_pos = true;+ hparams.use_alibi = true;}hparams.rope_type = llama_rope_type(&model);@@ -3971,11 +4032,13 @@ static void llm_load_vocab(// determine vocab type{- std::string tokenizer_name;+ std::string tokenizer_model;+ std::string tokenizer_pre;- ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_name);+ ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_model);+ ml.get_key(LLM_KV_TOKENIZER_PRE, tokenizer_pre, false);- if (tokenizer_name == "no_vocab") {+ if (tokenizer_model == "no_vocab") {vocab.type = LLAMA_VOCAB_TYPE_NONE;// default special tokens@@ -3989,7 +4052,7 @@ static void llm_load_vocab(vocab.linefeed_id = -1;return;- } else if (tokenizer_name == "llama") {+ } else if (tokenizer_model == "llama") {vocab.type = LLAMA_VOCAB_TYPE_SPM;// default special tokens@@ -4034,9 +4097,27 @@ static void llm_load_vocab(if (add_space_prefix_keyidx != -1) {vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);} // The default value of add_space_prefix is true.- } else if (tokenizer_name == "gpt2") {- vocab.type = LLAMA_VOCAB_TYPE_BPE;+ } else if (tokenizer_model == "bert") {+ vocab.type = LLAMA_VOCAB_TYPE_WPM;+ // default special tokens+ vocab.special_bos_id = -1;+ vocab.special_eos_id = -1;+ vocab.special_unk_id = 100;+ vocab.special_sep_id = 102;+ vocab.special_pad_id = 0;+ vocab.special_cls_id = 101;+ vocab.special_mask_id = 103;+ vocab.add_space_prefix = false;+ } else {+ if (tokenizer_model == "gpt2") {+ vocab.type = LLAMA_VOCAB_TYPE_BPE;+ } else {+ LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_model.c_str());+ LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__);+ vocab.type = LLAMA_VOCAB_TYPE_SPM;+ return;+ }// read bpe merges and populate bpe ranksconst int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());if (merges_keyidx == -1) {@@ -4070,23 +4151,50 @@ static void llm_load_vocab(vocab.special_pad_id = -1;vocab.special_cls_id = -1;vocab.special_mask_id = -1;- } else if (tokenizer_name == "bert") {- vocab.type = LLAMA_VOCAB_TYPE_WPM;+ }- // default special tokens- vocab.special_bos_id = -1;- vocab.special_eos_id = -1;- vocab.special_unk_id = 100;- vocab.special_sep_id = 102;- vocab.special_pad_id = 0;- vocab.special_cls_id = 101;- vocab.special_mask_id = 103;- vocab.add_space_prefix = false;+ // for now, only BPE models have pre-tokenizers+ if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {+ if (tokenizer_pre.empty()) {+ LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);+ LLAMA_LOG_WARN("%s: \n", __func__);+ LLAMA_LOG_WARN("%s: ************************************ \n", __func__);+ LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED! \n", __func__);+ LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL \n", __func__);+ LLAMA_LOG_WARN("%s: ************************************ \n", __func__);+ LLAMA_LOG_WARN("%s: \n", __func__);+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;+ } else if (+ tokenizer_pre == "default") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;+ } else if (+ tokenizer_pre == "llama3" ||+ tokenizer_pre == "llama-v3" ||+ tokenizer_pre == "llama-bpe") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;+ } else if (+ tokenizer_pre == "deepseek-llm") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;+ } else if (+ tokenizer_pre == "deepseek-coder") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER;+ } else if (+ tokenizer_pre == "falcon") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_FALCON;+ } else if (+ tokenizer_pre == "mpt") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_MPT;+ } else if (+ tokenizer_pre == "starcoder") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER;+ } else if (+ tokenizer_pre == "gpt-2") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;+ } else {+ throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));+ }} else {- LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_name.c_str());- LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__);-- vocab.type = LLAMA_VOCAB_TYPE_SPM;+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;}}@@ -5781,7 +5889,7 @@ static bool llm_load_tensors(// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callbackstatic int llama_model_load(const std::string & fname, llama_model & model, llama_model_params & params) {try {- llama_model_loader ml(fname, params.use_mmap, params.kv_overrides);+ llama_model_loader ml(fname, params.use_mmap, params.check_tensors, params.kv_overrides);model.hparams.vocab_only = params.vocab_only;@@ -5911,37 +6019,47 @@ static struct ggml_tensor * llm_build_inp_embd(static void llm_build_kv_store(struct ggml_context * ctx,const llama_hparams & hparams,+ const llama_cparams & cparams,const llama_kv_cache & kv,struct ggml_cgraph * graph,struct ggml_tensor * k_cur,struct ggml_tensor * v_cur,- int64_t n_ctx,int32_t n_tokens,int32_t kv_head,const llm_build_cb & cb,int64_t il) {+ const int64_t n_ctx = cparams.n_ctx;+const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();GGML_ASSERT(kv.size == n_ctx);- // compute the transposed [n_tokens, n_embd] V matrix- assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens);- struct ggml_tensor * v_cur_t = ggml_transpose(ctx, v_cur);- cb(v_cur_t, "v_cur_t", il);-struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k_l[il], n_tokens*n_embd_k_gqa,(ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa))*kv_head);cb(k_cache_view, "k_cache_view", il);- struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_v_gqa,- ( n_ctx)*ggml_element_size(kv.v_l[il]),- (kv_head)*ggml_element_size(kv.v_l[il]));+ // note: storing RoPE-ed version of K in the KV cache+ ggml_build_forward_expand(graph, ggml_cpy(ctx, k_cur, k_cache_view));++ assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens);++ struct ggml_tensor * v_cache_view = nullptr;++ if (cparams.flash_attn) {+ v_cache_view = ggml_view_1d(ctx, kv.v_l[il], n_tokens*n_embd_v_gqa,+ (kv_head)*ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa));+ } else {+ // note: the V cache is transposed when not using flash attention+ v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_v_gqa,+ ( n_ctx)*ggml_element_size(kv.v_l[il]),+ (kv_head)*ggml_element_size(kv.v_l[il]));++ v_cur = ggml_transpose(ctx, v_cur);+ }cb(v_cache_view, "v_cache_view", il);- // important: storing RoPE-ed version of K in the KV cache!- ggml_build_forward_expand(graph, ggml_cpy(ctx, k_cur, k_cache_view));- ggml_build_forward_expand(graph, ggml_cpy(ctx, v_cur_t, v_cache_view));+ ggml_build_forward_expand(graph, ggml_cpy(ctx, v_cur, v_cache_view));}static struct ggml_tensor * llm_build_norm(@@ -6161,11 +6279,11 @@ static struct ggml_tensor * llm_build_moe_ffn(return moe_out;}-// if max_alibi_bias > 0 then apply ALiBistatic struct ggml_tensor * llm_build_kqv(struct ggml_context * ctx,const llama_model & model,const llama_hparams & hparams,+ const llama_cparams & cparams,const llama_kv_cache & kv,struct ggml_cgraph * graph,struct ggml_tensor * wo,@@ -6173,12 +6291,12 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * q_cur,struct ggml_tensor * kq_mask,struct ggml_tensor * kq_pos,- int64_t n_ctx,int32_t n_tokens,int32_t n_kv,float kq_scale,const llm_build_cb & cb,int il) {+ const int64_t n_ctx = cparams.n_ctx;const int64_t n_head = hparams.n_head;const int64_t n_head_kv = hparams.n_head_kv;const int64_t n_embd_head_k = hparams.n_embd_head_k;@@ -6196,71 +6314,99 @@ static struct ggml_tensor * llm_build_kqv(0);cb(k, "k", il);- struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);- cb(kq, "kq", il);+ struct ggml_tensor * cur;- if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) {- // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs- // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847- ggml_mul_mat_set_prec(kq, GGML_PREC_F32);- }+ if (cparams.flash_attn) {+ GGML_UNUSED(model);+ GGML_UNUSED(n_ctx);- if (model.arch == LLM_ARCH_GROK) {- // need to do the following:- // multiply by attn_output_multiplyer of 0.08838834764831845- // and then :- // kq = 30 * tanh(kq / 30)- // before the softmax below+ // note: if this assert triggers, then some check has failed earlier+ // the idea is to detect during context creation that ALiBi would be used and disable Flash Attention+ GGML_ASSERT(kq_pos == nullptr && "ALiBi is not yet supported with Flash Attention");- //try from phi2- //ggml_mul_mat_set_prec(kq, GGML_PREC_F32);+ // split cached v into n_head heads (not transposed)+ struct ggml_tensor * v =+ ggml_view_3d(ctx, kv.v_l[il],+ n_embd_head_v, n_kv, n_head_kv,+ ggml_row_size(kv.v_l[il]->type, n_embd_k_gqa),+ ggml_row_size(kv.v_l[il]->type, n_embd_head_k),+ 0);+ cb(v, "v", il);- kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f));- kq = ggml_scale(ctx, kq, 30);- }+ cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale);++ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) {+ ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);+ }++ cur = ggml_reshape_2d(ctx, cur, n_embd_head_k*n_head, n_tokens);+ } else {+ struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);+ cb(kq, "kq", il);++ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) {+ // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs+ // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847+ ggml_mul_mat_set_prec(kq, GGML_PREC_F32);+ }++ if (model.arch == LLM_ARCH_GROK) {+ // need to do the following:+ // multiply by attn_output_multiplyer of 0.08838834764831845+ // and then :+ // kq = 30 * tanh(kq / 30)+ // before the softmax below++ //try from phi2+ //ggml_mul_mat_set_prec(kq, GGML_PREC_F32);++ kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f));+ kq = ggml_scale(ctx, kq, 30);+ }#if defined(GGML_USE_KOMPUTE)#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Kompute")#pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024")#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488")- if (hparams.f_max_alibi_bias > 0.0f) {- kq = ggml_scale(ctx, kq, kq_scale);- cb(kq, "kq_scaled", il);+ if (hparams.use_alibi) {+ kq = ggml_scale(ctx, kq, kq_scale);+ cb(kq, "kq_scaled", il);- kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, hparams.f_max_alibi_bias);- cb(kq, "kq_scaled_alibi", il);+ kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, hparams.f_max_alibi_bias);+ cb(kq, "kq_scaled_alibi", il);- kq = ggml_add(ctx, kq, kq_mask);- cb(kq, "kq_masked", il);+ kq = ggml_add(ctx, kq, kq_mask);+ cb(kq, "kq_masked", il);- kq = ggml_soft_max(ctx, kq);- cb(kq, "kq_soft_max", il);- } else+ kq = ggml_soft_max(ctx, kq);+ cb(kq, "kq_soft_max", il);+ } else#endif- {- kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_pos, kq_scale, hparams.f_max_alibi_bias);- cb(kq, "kq_soft_max_ext", il);- }+ {+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_pos, kq_scale, hparams.f_max_alibi_bias);+ cb(kq, "kq_soft_max_ext", il);+ }- GGML_ASSERT(kv.size == n_ctx);+ GGML_ASSERT(kv.size == n_ctx);- // split cached v into n_head heads- struct ggml_tensor * v =- ggml_view_3d(ctx, kv.v_l[il],- n_kv, n_embd_head_v, n_head_kv,- ggml_element_size(kv.v_l[il])*n_ctx,- ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head_v,- 0);- cb(v, "v", il);+ // split cached v into n_head heads+ struct ggml_tensor * v =+ ggml_view_3d(ctx, kv.v_l[il],+ n_kv, n_embd_head_v, n_head_kv,+ ggml_element_size(kv.v_l[il])*n_ctx,+ ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head_v,+ 0);+ cb(v, "v", il);- struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq);- cb(kqv, "kqv", il);+ struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq);+ cb(kqv, "kqv", il);- struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3);- cb(kqv_merged, "kqv_merged", il);+ struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3);+ cb(kqv_merged, "kqv_merged", il);- struct ggml_tensor * cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens);- cb(cur, "kqv_merged_cont", il);+ cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens);+ cb(cur, "kqv_merged_cont", il);+ }ggml_build_forward_expand(graph, cur);@@ -6280,6 +6426,7 @@ static struct ggml_tensor * llm_build_kv(struct ggml_context * ctx,const llama_model & model,const llama_hparams & hparams,+ const llama_cparams & cparams,const llama_kv_cache & kv,struct ggml_cgraph * graph,struct ggml_tensor * wo,@@ -6289,7 +6436,6 @@ static struct ggml_tensor * llm_build_kv(struct ggml_tensor * q_cur,struct ggml_tensor * kq_mask,struct ggml_tensor * kq_pos,- int64_t n_ctx,int32_t n_tokens,int32_t kv_head,int32_t n_kv,@@ -6303,12 +6449,12 @@ static struct ggml_tensor * llm_build_kv(ggml_build_forward_expand(graph, k_cur);ggml_build_forward_expand(graph, v_cur);- llm_build_kv_store(ctx, hparams, kv, graph, k_cur, v_cur, n_ctx, n_tokens, kv_head, cb, il);+ llm_build_kv_store(ctx, hparams, cparams, kv, graph, k_cur, v_cur, n_tokens, kv_head, cb, il);struct ggml_tensor * cur;- cur = llm_build_kqv(ctx, model, hparams, kv, graph, wo, wo_b,- q_cur, kq_mask, kq_pos, n_ctx, n_tokens, n_kv, kq_scale, cb, il);+ cur = llm_build_kqv(ctx, model, hparams, cparams, kv, graph, wo, wo_b,+ q_cur, kq_mask, kq_pos, n_tokens, n_kv, kq_scale, cb, il);cb(cur, "kqv_out", il);return cur;@@ -6350,6 +6496,8 @@ struct llm_build_context {const int32_t kv_head; // index of where we store new KV data in the cacheconst int32_t n_orig_ctx;+ const bool flash_attn;+const enum llama_pooling_type pooling_type;const enum llama_rope_type rope_type;@@ -6396,6 +6544,7 @@ struct llm_build_context {n_outputs (worst_case ? n_tokens : lctx.n_outputs),kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head),n_orig_ctx (cparams.n_yarn_orig_ctx),+ flash_attn (cparams.flash_attn),pooling_type (cparams.pooling_type),rope_type (hparams.rope_type),cb (cb),@@ -6510,15 +6659,31 @@ struct llm_build_context {ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*id));- ggml_tensor * view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],- nm, n_embd_v_gqa,- ggml_row_size(kv_self.v_l[il]->type, kv_self.size),- ggml_row_size(kv_self.v_l[il]->type, i));+ ggml_tensor * view_v_src;+ ggml_tensor * view_v_dst;++ if (flash_attn) {+ // NOTE: the V cache is not transposed when using flash attention+ view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],+ n_embd_v_gqa, nm,+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa),+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*i));- ggml_tensor * view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],- nm, n_embd_v_gqa,- ggml_row_size(kv_self.v_l[il]->type, kv_self.size),- ggml_row_size(kv_self.v_l[il]->type, id));+ view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],+ n_embd_v_gqa, nm,+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa),+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*id));+ } else {+ view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],+ nm, n_embd_v_gqa,+ ggml_row_size(kv_self.v_l[il]->type, kv_self.size),+ ggml_row_size(kv_self.v_l[il]->type, i));++ view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],+ nm, n_embd_v_gqa,+ ggml_row_size(kv_self.v_l[il]->type, kv_self.size),+ ggml_row_size(kv_self.v_l[il]->type, id));+ }ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_k_src, view_k_dst));ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_v_src, view_v_dst));@@ -6548,20 +6713,26 @@ struct llm_build_context {struct ggml_tensor * build_inp_KQ_mask(bool causal = true) {if (causal) {- lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_tokens);+ lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD));} else {- lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens);+ lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD));}cb(lctx.inp_KQ_mask, "KQ_mask", -1);ggml_set_input(lctx.inp_KQ_mask);- return lctx.inp_KQ_mask;+ return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_mask, GGML_TYPE_F16) : lctx.inp_KQ_mask;}- struct ggml_tensor * build_inp_KQ_pos() {- lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_kv);+ struct ggml_tensor * build_inp_KQ_pos(bool causal = true) {+ if (causal) {+ lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_kv);+ } else {+ // TODO: this will be needed for ALiBi-based BERT models+ // https://github.com/ggerganov/llama.cpp/pull/6826+ lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_tokens);+ }cb(lctx.inp_KQ_pos, "KQ_pos", -1);ggml_set_input(lctx.inp_KQ_pos);- return lctx.inp_KQ_pos;+ return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_pos, GGML_TYPE_F16) : lctx.inp_KQ_pos;}struct ggml_tensor * build_inp_mean() {@@ -6667,9 +6838,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -6807,9 +6978,9 @@ struct llm_build_context {cb(Qcur, "Qcur", il);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -6914,9 +7085,9 @@ struct llm_build_context {ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7034,9 +7205,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7159,9 +7330,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -7311,9 +7482,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,- model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,+ model.layers[il].wo, NULL,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7423,9 +7594,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7627,9 +7798,9 @@ struct llm_build_context {);cb(Vcur, "Vcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Q, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Q, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7723,9 +7894,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);cb(Qcur, "Qcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8016,9 +8187,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8147,14 +8318,15 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,- model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);} else {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,++ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}}@@ -8296,9 +8468,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8414,9 +8586,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8527,9 +8699,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8641,9 +8813,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8796,9 +8968,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -8913,9 +9085,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,- model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -9026,9 +9198,9 @@ struct llm_build_context {ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}struct ggml_tensor * sa_out = cur;@@ -9129,9 +9301,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9236,9 +9408,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9352,9 +9524,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9469,9 +9641,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9599,9 +9771,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9720,9 +9892,9 @@ struct llm_build_context {ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -9839,9 +10011,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -10129,9 +10301,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -10260,9 +10432,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, nullptr,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -10689,7 +10861,9 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {}}- if (hparams.need_kq_pos) {+ // ALiBi requires the KQ_pos tensor to provide the sequence position of each token in the batch+ // this allows to process multiple sequences in parallel with ALiBi-based models+ if (hparams.use_alibi) {const int64_t n_kv = kv_self.n;GGML_ASSERT(lctx.inp_KQ_pos);@@ -11071,7 +11245,7 @@ static int llama_decode_internal(// a heuristic, to avoid attending the full cache if it is not yet utilized// after enough generations, the benefit from this heuristic disappears// if we start defragmenting the cache, the benefit from this will be more important- kv_self.n = std::min(kv_self.size, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32)));+ kv_self.n = std::min(kv_self.size, std::max(256u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 256)));//kv_self.n = llama_kv_cache_cell_max(kv_self);}}@@ -11239,6 +11413,10 @@ static int llama_decode_internal(}}+ // Reset state for the next token before backend sync, to allow the CPU activities in the reset to+ // overlap with device computation.+ ggml_backend_sched_reset(lctx.sched);+return 0;}@@ -11264,7 +11442,9 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {// each move requires 6*n_layer tensors (see build_defrag)// - source view, destination view, copy operation// - x2 for keys and values- const uint32_t max_moves = LLAMA_MAX_NODES/(6*n_layer);+ //const uint32_t max_moves = LLAMA_MAX_NODES/(6*n_layer);+ // TODO: tmp fix https://github.com/ggerganov/llama.cpp/issues/6685#issuecomment-2057579516+ const uint32_t max_moves = (LLAMA_MAX_NODES - 2*n_layer)/(6*n_layer);// determine which KV cells to move where//@@ -11588,7 +11768,7 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {}case LLAMA_VOCAB_TYPE_BPE: {GGML_ASSERT(false);- return unicode_utf8_to_byte(token_data.text);+ return unicode_utf8_to_byte(token_data.text); // TODO: why is this here after GGML_ASSERT?}case LLAMA_VOCAB_TYPE_WPM: {GGML_ASSERT(false);@@ -11810,7 +11990,79 @@ struct llm_tokenizer_bpe {void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {int final_prev_index = -1;- auto word_collection = bpe_gpt2_preprocess(text);++ std::vector<std::string> word_collection;+ switch (vocab.type) {+ case LLAMA_VOCAB_TYPE_BPE:+ switch (vocab.type_pre) {+ case LLAMA_VOCAB_PRE_TYPE_LLAMA3:+ word_collection = unicode_regex_split(text, {+ // original regex from tokenizer.json+ //"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",++ // adapted: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2080233989+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",+ });+ break;+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:+ word_collection = unicode_regex_split(text, {+ "[\r\n]",+ "\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗA-Za-z𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+",+ "\\s?[!-/:-~!-/:-~‘-‟ -。]+",+ "\\s+$",+ "[一-龥ࠀ-一가-]+",+ "\\p{N}+",+ });+ break;+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:+ word_collection = unicode_regex_split(text, {+ "[\r\n]",+ "\\s?\\p{L}+",+ "\\s?\\p{P}+",+ "[一-龥ࠀ-一가-]+",+ "\\p{N}+",+ });+ break;+ case LLAMA_VOCAB_PRE_TYPE_FALCON:+ word_collection = unicode_regex_split(text, {+ "[\\p{P}\\$\\+<=>\\^~\\|]+",+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",+ "\\p{N}+",+ "[0-9][0-9][0-9]",+ });+ break;+ case LLAMA_VOCAB_PRE_TYPE_MPT:+ // TODO: MPT pre-tokenization regexes are unknown+ // the following are close, but not exact. run the following:+ // ./bin/test-tokenizer-0 ../models/ggml-vocab-mpt.gguf+ GGML_ASSERT("MPT pre-tokenization regexes are unknown - fixes needed");+ word_collection = unicode_regex_split(text, {+ "\\s?\\p{L}+",+ "\\s?\\p{P}+",+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",+ });+ break;+ case LLAMA_VOCAB_PRE_TYPE_STARCODER:+ case LLAMA_VOCAB_PRE_TYPE_GPT2:+ word_collection = unicode_regex_split(text, {+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",+ });+ break;+ default:+ // default regex for BPE tokenization pre-processing+ word_collection = unicode_regex_split(text, {+ "[\\p{P}\\$\\+<=>\\^~\\|]+",+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",+ "\\p{N}+",+ "[0-9][0-9][0-9]",+ });+ break;+ }+ break;+ default:+ GGML_ASSERT(false);+ break;+ }symbols_final.clear();@@ -11937,145 +12189,6 @@ private:work_queue.push(bigram);}- std::vector<std::string> bpe_gpt2_preprocess(const std::string & text) {- std::vector<std::string> bpe_words;- std::vector<std::string> bpe_encoded_words;-- std::string token = "";- // GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+- bool collecting_numeric = false;- bool collecting_letter = false;- bool collecting_special = false;- bool collecting_whitespace_lookahead = false;- bool collecting = false;-- std::vector<std::string> text_utf;- text_utf.reserve(text.size());- bpe_words.reserve(text.size());- bpe_encoded_words.reserve(text.size());-- const auto cpts = unicode_cpts_from_utf8(text);- for (size_t i = 0; i < cpts.size(); ++i)- text_utf.emplace_back(unicode_cpt_to_utf8(cpts[i]));-- for (int i = 0; i < (int)text_utf.size(); i++) {- const std::string & utf_char = text_utf[i];- bool split_condition = false;- int bytes_remain = text_utf.size() - i;- // forward backward lookups- const std::string & utf_char_next = (i + 1 < (int)text_utf.size()) ? text_utf[i + 1] : "";- const std::string & utf_char_next_next = (i + 2 < (int)text_utf.size()) ? text_utf[i + 2] : "";-- // handling contractions- if (!split_condition && bytes_remain >= 2) {- // 's|'t|'m|'d- if (utf_char == "\'" && (utf_char_next == "s" || utf_char_next == "t" || utf_char_next == "m" || utf_char_next == "d")) {- split_condition = true;- }- if (split_condition) {- if (token.size()) {- bpe_words.emplace_back(token); // push previous content as token- }- token = utf_char + utf_char_next;- bpe_words.emplace_back(token);- token = "";- i++;- continue;- }- }- if (!split_condition && bytes_remain >= 3) {- // 're|'ve|'ll- if (utf_char == "\'" && (- (utf_char_next == "r" && utf_char_next_next == "e") ||- (utf_char_next == "v" && utf_char_next_next == "e") ||- (utf_char_next == "l" && utf_char_next_next == "l"))- ) {- split_condition = true;- }- if (split_condition) {- // current token + next token can be defined- if (token.size()) {- bpe_words.emplace_back(token); // push previous content as token- }- token = utf_char + utf_char_next + utf_char_next_next;- bpe_words.emplace_back(token); // the contraction- token = "";- i += 2;- continue;- }- }-- if (!split_condition && !collecting) {- if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) {- collecting_letter = true;- collecting = true;- }- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {- collecting_numeric = true;- collecting = true;- }- else if (- ((unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (unicode_cpt_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) ||- (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE)- ) {- collecting_special = true;- collecting = true;- }- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) {- collecting_whitespace_lookahead = true;- collecting = true;- }- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) {- split_condition = true;- }- }- else if (!split_condition && collecting) {- if (collecting_letter && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER) {- split_condition = true;- }- else if (collecting_numeric && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) {- split_condition = true;- }- else if (collecting_special && (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) {- split_condition = true;- }- else if (collecting_whitespace_lookahead && (unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {- split_condition = true;- }- }-- if (utf_char_next == "") {- split_condition = true; // final- token += utf_char;- }-- if (split_condition) {- if (token.size()) {- bpe_words.emplace_back(token);- }- token = utf_char;- collecting = false;- collecting_letter = false;- collecting_numeric = false;- collecting_special = false;- collecting_whitespace_lookahead = false;- }- else {- token += utf_char;- }- }-- for (std::string & word : bpe_words) {- std::string encoded_token = "";- for (char & c : word) {- encoded_token += unicode_byte_to_utf8(c);- }- bpe_encoded_words.emplace_back(encoded_token);- }-- return bpe_encoded_words;- }-const llama_vocab & vocab;std::vector<llm_symbol> symbols;@@ -12395,7 +12508,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &} break;case LLAMA_VOCAB_TYPE_BPE:{- if (add_special && vocab.special_add_bos == 1) {+ if (add_special && vocab.special_add_bos != 0) {GGML_ASSERT(vocab.special_bos_id != -1);output.push_back(vocab.special_bos_id);}@@ -14167,14 +14280,20 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n}static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t nrows, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {- std::mutex mutex;- int64_t counter = 0;- size_t new_size = 0;if (nthread < 2) {// single-thread- return ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);+ size_t new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);+ if (!ggml_validate_row_data(new_type, new_data, new_size)) {+ throw std::runtime_error("quantized data validation failed");+ }+ return new_size;}- auto compute = [&mutex, &counter, &new_size, new_type, f32_data, new_data, chunk_size,++ std::mutex mutex;+ int64_t counter = 0;+ size_t new_size = 0;+ bool valid = true;+ auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,nrows, n_per_row, imatrix]() {const int64_t nrows_per_chunk = chunk_size / n_per_row;size_t local_size = 0;@@ -14189,7 +14308,17 @@ static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const floa}lock.unlock();const int64_t this_nrow = std::min(nrows - first_row, nrows_per_chunk);- local_size += ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);+ size_t this_size = ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);+ local_size += this_size;++ // validate the quantized data+ const size_t row_size = ggml_row_size(new_type, n_per_row);+ void * this_data = (char *) new_data + first_row * row_size;+ if (!ggml_validate_row_data(new_type, this_data, this_size)) {+ std::unique_lock<std::mutex> lock(mutex);+ valid = false;+ break;+ }}};for (int it = 0; it < nthread - 1; ++it) {@@ -14198,6 +14327,9 @@ static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const floacompute();for (auto & w : workers) { w.join(); }workers.clear();+ if (!valid) {+ throw std::runtime_error("quantized data validation failed");+ }return new_size;}@@ -14261,7 +14393,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sauto v = (std::vector<llama_model_kv_override>*)params->kv_overrides;kv_overrides = v->data();}- llama_model_loader ml(fname_inp, use_mmap, kv_overrides);+ llama_model_loader ml(fname_inp, use_mmap, /*check_tensors*/ true, kv_overrides);ml.init_mappings(false); // no prefetchingllama_model model;@@ -14299,11 +14431,13 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sfor (auto & o : overrides) {if (o.key[0] == 0) break;if (o.tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {- gguf_set_val_f32(ctx_out, o.key, o.float_value);+ gguf_set_val_f32(ctx_out, o.key, o.val_f64);} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_INT) {- gguf_set_val_i32(ctx_out, o.key, o.int_value);+ gguf_set_val_i32(ctx_out, o.key, o.val_i64);} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {- gguf_set_val_bool(ctx_out, o.key, o.bool_value);+ gguf_set_val_bool(ctx_out, o.key, o.val_bool);+ } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_STR) {+ gguf_set_val_str(ctx_out, o.key, o.val_str);} else {LLAMA_LOG_WARN("%s: unknown KV override type for key %s\n", __func__, o.key);}@@ -14622,7 +14756,7 @@ static int llama_apply_lora_from_file_internal(std::unique_ptr<llama_model_loader> ml;if (path_base_model) {LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model);- ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ nullptr));+ ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*check_tensors*/ false, /*kv_overrides*/ nullptr));ml->init_mappings(/*prefetch*/ false); // no prefetching}@@ -14881,12 +15015,13 @@ struct llama_model_params llama_model_default_params() {/*.vocab_only =*/ false,/*.use_mmap =*/ true,/*.use_mlock =*/ false,+ /*.check_tensors =*/ false,};- if (llamafile_has_metal()) {- // // note: we usually have plenty of VRAM, so by default offload all layers to the GPU- // result.n_gpu_layers = 999; // [jart] let llamafile/gpu.c handle this- }+#ifdef GGML_USE_METAL // [jart] let llamafile/gpu.c handle this+ // note: we usually have plenty of VRAM, so by default offload all layers to the GPU+ result.n_gpu_layers = 999;+#endifreturn result;}@@ -14917,6 +15052,7 @@ struct llama_context_params llama_context_default_params() {/*.logits_all =*/ false,/*.embeddings =*/ false,/*.offload_kqv =*/ true,+ /*.flash_attn =*/ false,/*.abort_callback =*/ nullptr,/*.abort_callback_data =*/ nullptr,};@@ -15078,6 +15214,7 @@ struct llama_context * llama_new_context_with_model(cparams.defrag_thold = params.defrag_thold;cparams.embeddings = params.embeddings;cparams.offload_kqv = params.offload_kqv;+ cparams.flash_attn = params.flash_attn;cparams.pooling_type = params.pooling_type;cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;@@ -15085,12 +15222,20 @@ struct llama_context * llama_new_context_with_model(cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;// this is necessary due to kv_self.n being padded later during inference- cparams.n_ctx = GGML_PAD(cparams.n_ctx, 32);+ cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);// with causal attention, the batch size is limited by the context sizecparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;- cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);+ // the batch has to be at least GGML_KQ_MASK_PAD because we will be padding the KQ_mask+ // this is required by GPU kernels in order to avoid out-of-bounds accesses (e.g. ggml_flash_attn_ext)+ // ref: https://github.com/ggerganov/llama.cpp/pull/5021+ if (cparams.n_batch < GGML_KQ_MASK_PAD) {+ LLAMA_LOG_WARN("%s: n_batch is less than GGML_KQ_MASK_PAD - increasing to %d\n", __func__, GGML_KQ_MASK_PAD);+ cparams.n_batch = GGML_KQ_MASK_PAD;+ }++ cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);cparams.n_yarn_orig_ctx = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :hparams.n_yarn_orig_ctx != 0 ? hparams.n_yarn_orig_ctx :@@ -15122,6 +15267,23 @@ struct llama_context * llama_new_context_with_model(}}+ if (cparams.flash_attn && hparams.use_alibi) {+ LLAMA_LOG_WARN("%s: flash_attn is not yet compatible with ALiBi - forcing off\n", __func__);+ cparams.flash_attn = false;+ }++ if (cparams.flash_attn && model->arch == LLM_ARCH_GROK) {+ LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__);+ cparams.flash_attn = false;+ }++// #ifdef GGML_USE_HIPBLAS [jart]+ if (cparams.flash_attn && llamafile_has_amd_gpu()) {+ LLAMA_LOG_WARN("%s: flash_attn is not yet compatible with AMD GPUs builds - forcing off\n", __func__);+ cparams.flash_attn = false;+ }+// #endif+if (params.seed == LLAMA_DEFAULT_SEED) {params.seed = _rand64();}@@ -15129,6 +15291,7 @@ struct llama_context * llama_new_context_with_model(LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);+ LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn);LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);@@ -15247,7 +15410,7 @@ if (llamafile_has_metal()) {}ctx->backends.push_back(ctx->backend_cpu);- if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, kv_size, cparams.offload_kqv)) {+ if (!llama_kv_cache_init(ctx->kv_self, ctx, type_k, type_v, kv_size, cparams.offload_kqv)) {LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__);llama_free(ctx);return nullptr;@@ -15848,6 +16011,7 @@ size_t llama_state_get_size(const struct llama_context * ctx) {const size_t s_kv_head = sizeof(uint32_t);const size_t s_kv_size = sizeof(uint32_t);const size_t s_kv_used = sizeof(uint32_t);+ const size_t s_v_trans = sizeof(uint32_t);const size_t s_kv = ctx->kv_self.total_size();const size_t s_kv_cell = sizeof(llama_pos) + sizeof(size_t) + cparams.n_seq_max*sizeof(llama_seq_id);const size_t s_kv_cells = ctx->kv_self.size * s_kv_cell;@@ -15865,10 +16029,14 @@ size_t llama_state_get_size(const struct llama_context * ctx) {+ s_kv_head+ s_kv_size+ s_kv_used+ + s_v_trans+ s_kv+ s_kv_cells);+ // on session change it is very likely that the state size has changed - so we need to update this function+ static_assert(LLAMA_SESSION_VERSION == 6, "So you just bumped the session version - good. But did you remember to update llama_state_get_size?");+return s_total;}@@ -16014,11 +16182,13 @@ static void llama_state_get_data_internal(struct llama_context * ctx, llama_dataconst uint32_t kv_size = kv_self.size;const size_t kv_buf_size = kv_self.total_size() / (kv_size ? kv_size : 1) * kv_head;const uint32_t kv_used = kv_self.used;+ const uint32_t v_trans = kv_self.v_trans ? 1 : 0;data_ctx->write(&kv_buf_size, sizeof(kv_buf_size));data_ctx->write(&kv_head, sizeof(kv_head));data_ctx->write(&kv_size, sizeof(kv_size));data_ctx->write(&kv_used, sizeof(kv_used));+ data_ctx->write(&v_trans, sizeof(v_trans));if (kv_buf_size) {const size_t pre_kv_buf_size = data_ctx->get_size_written();@@ -16031,7 +16201,7 @@ static void llama_state_get_data_internal(struct llama_context * ctx, llama_dataggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size());data_ctx->write(tmp_buf.data(), tmp_buf.size());- if (kv_self.recurrent) {+ if (kv_self.recurrent || !kv_self.v_trans) {// v is contiguous for recurrent models// TODO: use other tensors for state models than k and vconst size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head);@@ -16164,11 +16334,15 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {uint32_t kv_head;uint32_t kv_size;uint32_t kv_used;+ uint32_t v_trans;memcpy(&kv_buf_size, inp, sizeof(kv_buf_size)); inp += sizeof(kv_buf_size);memcpy(&kv_head, inp, sizeof(kv_head)); inp += sizeof(kv_head);memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size);memcpy(&kv_used, inp, sizeof(kv_used)); inp += sizeof(kv_used);+ memcpy(&v_trans, inp, sizeof(v_trans)); inp += sizeof(v_trans);++ GGML_ASSERT(kv_self.v_trans == (bool) v_trans); // incompatible V transpositionif (kv_self.size != kv_size) {// the KV cache needs to be big enough to load all the KV cells from the saved state@@ -16178,6 +16352,8 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {__func__, kv_head, kv_size, kv_self.size);}+ llama_kv_cache_clear(ctx);+if (kv_buf_size) {const size_t pre_kv_buf_size = inp - src;@@ -16189,7 +16365,7 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size);inp += k_size;- if (kv_self.recurrent) {+ if (kv_self.recurrent || !kv_self.v_trans) {// v is contiguous for recurrent models// TODO: use other tensors for state models than k and vconst size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head);@@ -16211,8 +16387,6 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {GGML_ASSERT(kv_buf_size == inp - src - pre_kv_buf_size);}- llama_kv_cache_clear(ctx);-ctx->kv_self.head = kv_head;ctx->kv_self.used = kv_used;@@ -16472,28 +16646,49 @@ static size_t llama_state_seq_get_data_internal(struct llama_context * ctx, llam}}- // For the values, they are transposed, so we also need the element size and get the element ranges from each row- const uint32_t kv_size = kv_self.size;- for (int il = 0; il < (int)n_layer; ++il) {- // Write value type- const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;- data_ctx.write(&v_type_i, sizeof(v_type_i));+ // TODO: simplify, reduce copy-paste+ if (!kv_self.v_trans) {+ for (int il = 0; il < (int)n_layer; ++il) {+ // Write value type+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;+ data_ctx.write(&v_type_i, sizeof(v_type_i));- // Write element size- const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);- data_ctx.write(&v_size_el, sizeof(v_size_el));+ // Write row size of value+ const size_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa);+ data_ctx.write(&v_size_row, sizeof(v_size_row));- // For each row, we get the element values of each cell- for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {- // Read each range of cells of v_size_el length each into tmp_buf and write out+ // Read each range of cells of v_size length each into tmp_buf and write outfor (const auto & range : cell_ranges) {const size_t range_size = range.second - range.first;- const size_t src_offset = (range.first + j * kv_size) * v_size_el;- tmp_buf.resize(range_size * v_size_el);- ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), src_offset, tmp_buf.size());+ tmp_buf.resize(range_size * v_size_row);+ ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), range.first * v_size_row, range_size * v_size_row);data_ctx.write(tmp_buf.data(), tmp_buf.size());}}+ } else {+ // For the values, they are transposed, so we also need the element size and get the element ranges from each row+ const uint32_t kv_size = kv_self.size;+ for (int il = 0; il < (int)n_layer; ++il) {+ // Write value type+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;+ data_ctx.write(&v_type_i, sizeof(v_type_i));++ // Write element size+ const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);+ data_ctx.write(&v_size_el, sizeof(v_size_el));++ // For each row, we get the element values of each cell+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {+ // Read each range of cells of v_size_el length each into tmp_buf and write out+ for (const auto & range : cell_ranges) {+ const size_t range_size = range.second - range.first;+ const size_t src_offset = (range.first + j * kv_size) * v_size_el;+ tmp_buf.resize(range_size * v_size_el);+ ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), src_offset, tmp_buf.size());+ data_ctx.write(tmp_buf.data(), tmp_buf.size());+ }+ }+ }}return data_ctx.get_size_written();@@ -16618,41 +16813,75 @@ size_t llama_state_seq_set_data(struct llama_context * ctx, const uint8_t * src,}}- // For each layer, read the values for each cell (transposed)- for (int il = 0; il < (int)n_layer; ++il) {- // Read type of value- int32_t v_type_i_ref;- memcpy(&v_type_i_ref, inp, sizeof(v_type_i_ref));- inp += sizeof(v_type_i_ref);- const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;- if (v_type_i != v_type_i_ref) {- llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);- LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);- return 0;- }+ // TODO: simplify, reduce copy-paste+ if (!kv_self.v_trans) {+ for (int il = 0; il < (int)n_layer; ++il) {+ // Read type of value+ int32_t v_type_i_ref;+ memcpy(&v_type_i_ref, inp, sizeof(v_type_i_ref));+ inp += sizeof(v_type_i_ref);+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;+ if (v_type_i != v_type_i_ref) {+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);+ LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);+ return 0;+ }- // Read element size of value- size_t v_size_el_ref;- memcpy(&v_size_el_ref, inp, sizeof(v_size_el_ref));- inp += sizeof(v_size_el_ref);- const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);- if (v_size_el != v_size_el_ref) {- llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);- LLAMA_LOG_ERROR("%s: mismatched value element size (%zu != %zu, layer %d)\n", __func__, v_size_el, v_size_el_ref, il);- return 0;- }+ // Read row size of value+ size_t v_size_row_ref;+ memcpy(&v_size_row_ref, inp, sizeof(v_size_row_ref));+ inp += sizeof(v_size_row_ref);+ const size_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa);+ if (v_size_row != v_size_row_ref) {+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);+ LLAMA_LOG_ERROR("%s: mismatched value row size (%zu != %zu, layer %d)\n", __func__, v_size_row, v_size_row_ref, il);+ return 0;+ }- if (cell_count) {- // For each row in the transposed matrix, read the values for the whole cell range- for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {- const size_t dst_offset = (kv_head + j * kv_size) * v_size_el;- ggml_backend_tensor_set(kv_self.v_l[il], inp, dst_offset, cell_count * v_size_el);- inp += cell_count * v_size_el;+ if (cell_count) {+ // Read and set the values for the whole cell range+ ggml_backend_tensor_set(kv_self.v_l[il], inp, kv_head * v_size_row, cell_count * v_size_row);+ inp += cell_count * v_size_row;+ }+ }+ } else {+ // For each layer, read the values for each cell (transposed)+ for (int il = 0; il < (int)n_layer; ++il) {+ // Read type of value+ int32_t v_type_i_ref;+ memcpy(&v_type_i_ref, inp, sizeof(v_type_i_ref));+ inp += sizeof(v_type_i_ref);+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;+ if (v_type_i != v_type_i_ref) {+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);+ LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);+ return 0;+ }++ // Read element size of value+ size_t v_size_el_ref;+ memcpy(&v_size_el_ref, inp, sizeof(v_size_el_ref));+ inp += sizeof(v_size_el_ref);+ const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);+ if (v_size_el != v_size_el_ref) {+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);+ LLAMA_LOG_ERROR("%s: mismatched value element size (%zu != %zu, layer %d)\n", __func__, v_size_el, v_size_el_ref, il);+ return 0;+ }++ if (cell_count) {+ // For each row in the transposed matrix, read the values for the whole cell range+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {+ const size_t dst_offset = (kv_head + j * kv_size) * v_size_el;+ ggml_backend_tensor_set(kv_self.v_l[il], inp, dst_offset, cell_count * v_size_el);+ inp += cell_count * v_size_el;+ }}}}const size_t nread = inp - src;+return nread;}@@ -17449,9 +17678,9 @@ const char * llama_print_system_info(void) {s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | ";s += "MATMUL_INT8 = " + std::to_string(ggml_cpu_has_matmul_int8()) + " | ";#ifdef GGML_USE_LLAMAFILE- s += "LAMMAFILE = 1 | ";+ s += "LLAMAFILE = 1 | ";#else- s += "LAMMAFILE = 0 | ";+ s += "LLAMAFILE = 0 | ";#endifreturn s.c_str();@@ -17495,7 +17724,7 @@ const std::vector<std::pair<std::string, struct ggml_tensor *>> & llama_internalvoid llama_log_set(ggml_log_callback log_callback, void * user_data) {g_state.log_callback = log_callback ? log_callback : llama_log_callback_default;g_state.log_callback_user_data = user_data;-#ifdef GGML_USE_METAL+#ifdef GGML_USE_METAL // [jart] do not wantggml_backend_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);#endif}diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex d49b8dd2..ff608594 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -42,7 +42,7 @@#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN-#define LLAMA_SESSION_VERSION 5+#define LLAMA_SESSION_VERSION 6#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ#define LLAMA_STATE_SEQ_VERSION 1@@ -71,6 +71,18 @@ extern "C" {LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece};+ // pre-tokenization types+ enum llama_vocab_pre_type {+ LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,+ LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,+ LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,+ LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,+ LLAMA_VOCAB_PRE_TYPE_FALCON = 4,+ LLAMA_VOCAB_PRE_TYPE_MPT = 5,+ LLAMA_VOCAB_PRE_TYPE_STARCODER = 6,+ LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,+ };+// note: these values should be synchronized with ggml_rope// TODO: maybe move this enum to ggml.h (ggml_rope_type)enum llama_rope_type {@@ -198,15 +210,19 @@ extern "C" {LLAMA_KV_OVERRIDE_TYPE_INT,LLAMA_KV_OVERRIDE_TYPE_FLOAT,LLAMA_KV_OVERRIDE_TYPE_BOOL,+ LLAMA_KV_OVERRIDE_TYPE_STR,};struct llama_model_kv_override {- char key[128];enum llama_model_kv_override_type tag;++ char key[128];+union {- int64_t int_value;- double float_value;- bool bool_value;+ int64_t val_i64;+ double val_f64;+ bool val_bool;+ char val_str[128];};};@@ -235,9 +251,10 @@ extern "C" {const struct llama_model_kv_override * kv_overrides;// Keep the booleans together to avoid misalignment during copy-by-value.- bool vocab_only; // only load the vocabulary, no weights- bool use_mmap; // use mmap if possible- bool use_mlock; // force system to keep model in RAM+ bool vocab_only; // only load the vocabulary, no weights+ bool use_mmap; // use mmap if possible+ bool use_mlock; // force system to keep model in RAM+ bool check_tensors; // validate model tensor data};struct llama_context_params {@@ -273,6 +290,7 @@ extern "C" {bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)bool embeddings; // if true, extract embeddings (together with logits)bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU+ bool flash_attn; // whether to use flash attention// Abort callback// if it returns true, execution of llama_decode() will be aborted@@ -528,7 +546,7 @@ extern "C" {// Returns the number of used KV cells (i.e. have at least one sequence assigned to them)LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx);- // Clear the KV cache+ // Clear the KV cache - both cell info is erased and KV data is zeroedLLAMA_API void llama_kv_cache_clear(struct llama_context * ctx);diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cppindex 766fd813..f7202ee9 100644--- a/llama.cpp/llava/llava-cli.cpp+++ b/llama.cpp/llava/llava-cli.cpp@@ -117,7 +117,6 @@ struct llava_context {};struct llava_context * volatile g_ctx;-static void sigint_handler(int signo) {if (signo == SIGINT) {printf("\n");@@ -127,11 +126,11 @@ static void sigint_handler(int signo) {}static void show_additional_info(int /*argc*/, char ** argv) {- LOG_TEE("\n example usage: %s -m <llava-v1.5-7b/ggml-model-q5_k.gguf> --mmproj <llava-v1.5-7b/mmproj-model-f16.gguf> --image <path/to/an/image.jpg> [--temp 0.1] [-p \"describe the image in detail.\"]\n", argv[0]);+ LOG_TEE("\n example usage: %s -m <llava-v1.5-7b/ggml-model-q5_k.gguf> --mmproj <llava-v1.5-7b/mmproj-model-f16.gguf> --image <path/to/an/image.jpg> --image <path/to/another/image.jpg> [--temp 0.1] [-p \"describe the image in detail.\"]\n", argv[0]);LOG_TEE(" note: a lower temperature value like 0.1 is recommended for better quality.\n");}-static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_params * params) {+static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_params * params, const std::string & fname) {// load and preprocess the imagellava_image_embed * embed = NULL;@@ -147,9 +146,9 @@ static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_para}params->prompt = remove_image_from_prompt(prompt);} else {- embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, params->image.c_str());+ embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, fname.c_str());if (!embed) {- LOG_TEE("%s: is %s really an image file?\n", __func__, params->image.c_str());+ fprintf(stderr, "%s: is %s really an image file?\n", __func__, fname.c_str());return NULL;}}@@ -221,17 +220,7 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_printf("\n");}--static struct llava_context * llava_init(gpt_params * params) {- const char * clip_path = params->mmproj.c_str();-- auto prompt = params->prompt;- if (prompt.empty()) {- prompt = "describe the image in detail.";- }-- auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);-+static struct llama_model * llava_init(gpt_params * params) {llama_backend_init();llama_numa_init(params->numa);@@ -242,6 +231,19 @@ static struct llava_context * llava_init(gpt_params * params) {LOG_TEE("%s: error: unable to load model\n" , __func__);return NULL;}+ return model;+}++static struct llava_context * llava_init_context(gpt_params * params, llama_model * model) {+ const char * clip_path = params->mmproj.c_str();++ auto prompt = params->prompt;+ if (prompt.empty()) {+ prompt = "describe the image in detail.";+ }++ auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);+llama_context_params ctx_params = llama_context_params_from_gpt_params(*params);ctx_params.n_ctx = params->n_ctx < 2048 ? 2048 : params->n_ctx; // we need a longer context size to process image embeddings@@ -272,14 +274,14 @@ static void llava_free(struct llava_context * ctx_llava) {llama_backend_free();}-int llava_cli(int argc, char ** argv, gpt_params * params) {+static void llama_log_callback_logTee(ggml_log_level level, const char * text, void * user_data) {+ (void) level;+ (void) user_data;+ LOG_TEE("%s", text);+}- auto ctx_llava = llava_init(params);- if (ctx_llava == NULL) {- fprintf(stderr, "%s: error: failed to init llava\n", __func__);- return 1;- }- g_ctx = ctx_llava;+int llava_cli(int argc, char ** argv, gpt_params & params) {+ ggml_time_init();struct sigaction sa;sa.sa_handler = sigint_handler;@@ -287,17 +289,42 @@ int llava_cli(int argc, char ** argv, gpt_params * params) {sa.sa_flags = 0;sigaction(SIGINT, &sa, NULL);- auto image_embed = load_image(ctx_llava, params);- if (!image_embed) {+#ifndef LOG_DISABLE_LOGS+ log_set_target(log_filename_generator("llava", "log"));+ LOG_TEE("Log start\n");+ log_dump_cmdline(argc, argv);+ llama_log_set(llama_log_callback_logTee, nullptr);+#endif // LOG_DISABLE_LOGS++ if (params.mmproj.empty() || (params.image.empty() && !prompt_contains_image(params.prompt))) {+ gpt_print_usage(argc, argv, params);+ show_additional_info(argc, argv);+ return 1;+ }+ auto model = llava_init(¶ms);+ if (model == NULL) {+ fprintf(stderr, "%s: error: failed to init llava model\n", __func__);return 1;}- // process the prompt- process_prompt(ctx_llava, image_embed, params, params->prompt);+ for (auto & image : params.image) {+ auto ctx_llava = llava_init_context(¶ms, model);- llama_print_timings(ctx_llava->ctx_llama);+ auto image_embed = load_image(ctx_llava, ¶ms, image);+ if (!image_embed) {+ std::cerr << "error: failed to load image " << image << ". Terminating\n\n";+ return 1;+ }++ // process the prompt+ process_prompt(ctx_llava, image_embed, ¶ms, params.prompt);++ llama_print_timings(ctx_llava->ctx_llama);+ llava_image_embed_free(image_embed);+ ctx_llava->model = NULL;+ llava_free(ctx_llava);+ }+ llama_free_model(model);- llava_image_embed_free(image_embed);- llava_free(ctx_llava);return 0;}diff --git a/llama.cpp/llava/llava.h b/llama.cpp/llava/llava.hindex 47b6784c..d6d1272f 100644--- a/llama.cpp/llava/llava.h+++ b/llama.cpp/llava/llava.h@@ -46,7 +46,7 @@ LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed);/** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past);-LLAVA_API int llava_cli(int argc, char ** argv, gpt_params * params);+LLAVA_API int llava_cli(int argc, char ** argv, gpt_params & params);#ifdef __cplusplus}diff --git a/llama.cpp/log.h b/llama.cpp/log.hindex a7448024..3c2d077c 100644--- a/llama.cpp/log.h+++ b/llama.cpp/log.h@@ -235,7 +235,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::// INTERNAL, DO NOT USE// USE LOG() INSTEAD//-#if !defined(_MSC_VER) or defined(__INTEL_LLVM_COMPILER)+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER)#define LOG_IMPL(str, ...) \do { \if (LOG_TARGET != nullptr) \@@ -258,7 +258,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::// INTERNAL, DO NOT USE// USE LOG_TEE() INSTEAD//-#if !defined(_MSC_VER) or defined(__INTEL_LLVM_COMPILER)+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER)#define LOG_TEE_IMPL(str, ...) \do { \if (LOG_TARGET != nullptr) \diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 84388b1e..988438bf 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -211,7 +211,7 @@ int main(int argc, char ** argv) {if (!params.mmproj.empty() &&(!params.image.empty() ||params.prompt.find("<img src=\"") != std::string::npos)) {- return llava_cli(argc, argv, ¶ms);+ return llava_cli(argc, argv, params);}// TODO: Dump params ?@@ -404,7 +404,7 @@ int main(int argc, char ** argv) {log_tostr(embd_inp.empty()), n_matching_session_tokens, embd_inp.size(), session_tokens.size(), embd_inp.size());// if we will use the cache for the full prompt without reaching the end of the cache, force- // reevaluation of the last token token to recalculate the cached logits+ // reevaluation of the last token to recalculate the cached logitsif (!embd_inp.empty() && n_matching_session_tokens == embd_inp.size() && session_tokens.size() > embd_inp.size()) {LOGLN("recalculate the cached logits (do): session_tokens.resize( %zu )", embd_inp.size() - 1);@@ -590,7 +590,6 @@ int main(int argc, char ** argv) {}struct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams);- bool should_show_special_tokens = sparams.grammar.empty();while ((n_remain != 0 && !is_antiprompt) || params.interactive) {// predict@@ -804,8 +803,7 @@ int main(int argc, char ** argv) {// display textif (input_echo && display) {for (auto id : embd) {- const std::string token_str =- llama_token_to_piece(ctx, id, should_show_special_tokens);+ const std::string token_str = llama_token_to_piece(ctx, id);printf("%s", token_str.c_str());if (embd.size() > 1) {@@ -971,7 +969,7 @@ int main(int argc, char ** argv) {for (size_t i = original_size; i < embd_inp.size(); ++i) {const llama_token token = embd_inp[i];output_tokens.push_back(token);- output_ss << llama_token_to_piece(ctx, token, should_show_special_tokens);+ output_ss << llama_token_to_piece(ctx, token);}n_remain -= line_inp.size();diff --git a/llama.cpp/perplexity/perplexity.cpp b/llama.cpp/perplexity/perplexity.cppindex e06e5071..d192b9de 100644--- a/llama.cpp/perplexity/perplexity.cpp+++ b/llama.cpp/perplexity/perplexity.cpp@@ -218,17 +218,22 @@ static void process_logits(std::ostream& out, int n_vocab, const float * logits,}struct kl_divergence_result {- double sum_nll = 0;- double sum_nll2 = 0;- double sum_kld = 0;- double sum_kld2 = 0;- double sum_nll_diff = 0;- double sum_nll_diff2 = 0;- size_t n_same_top = 0;- size_t count = 0;+ double sum_nll = 0.0;+ double sum_nll2 = 0.0;+ double sum_nll_base = 0.0;+ double sum_nll_base2 = 0.0;+ double sum_nll_nll_base = 0.0;+ double sum_kld = 0.0;+ double sum_kld2 = 0.0;+ double sum_p_diff = 0.0;+ double sum_p_diff2 = 0.0;+ double sum_p_diff4 = 0.0;+ float max_p_diff = 0.0f;+ size_t n_same_top = 0.0;+ size_t count = 0.0;};-static double log_softmax(int n_vocab, const float * logits, const uint16_t * base_log_prob, int tok, kl_divergence_result & kld) {+static std::pair<double, float> log_softmax(int n_vocab, const float * logits, const uint16_t * base_log_prob, int tok, kl_divergence_result & kld) {float max_logit = logits[0];int imax = 0;for (int i = 1; i < n_vocab; ++i) {@@ -246,12 +251,17 @@ static double log_softmax(int n_vocab, const float * logits, const uint16_t * baconst float scale = d[0];const float min_log_prob = d[1];base_log_prob += 4;- float nll = max_logit + log_sum_exp - logits[tok];++ const float nll = max_logit + log_sum_exp - logits[tok];kld.sum_nll += nll;kld.sum_nll2 += nll*nll;- nll += (scale*base_log_prob[tok] + min_log_prob);- kld.sum_nll_diff += nll;- kld.sum_nll_diff2 += nll*nll;++ const float nll_base = -(scale*base_log_prob[tok] + min_log_prob);+ kld.sum_nll_base += nll_base;+ kld.sum_nll_base2 += nll_base*nll_base;++ kld.sum_nll_nll_base += nll*nll_base;+max_logit += log_sum_exp;double sum = 0;int imax_base = -1;@@ -271,34 +281,50 @@ static double log_softmax(int n_vocab, const float * logits, const uint16_t * bakld.sum_kld2 += sum*sum;++kld.count;if (imax == imax_base) ++kld.n_same_top;- return sum;++ const float p_base = expf(-nll_base);+ const float p = expf(-nll);+ const float p_diff = p - p_base;+ kld.sum_p_diff += p_diff;+ const double p_diff2 = p_diff*p_diff;+ kld.sum_p_diff2 += p_diff2;+ kld.sum_p_diff4 += p_diff2*p_diff2;+ kld.max_p_diff = std::max(kld.max_p_diff, std::fabs(p_diff));++ return std::make_pair(sum, p_diff);}static void process_logits(int n_vocab, const float * logits, const int * tokens, int n_token,std::vector<std::thread> & workers, const std::vector<uint16_t> & base_log_probs, kl_divergence_result & kld,- float * kld_values) {+ float * kld_values, float * p_diff_values) {std::mutex mutex;const int nv = 2*((n_vocab + 1)/2) + 4;int counter = 0;- auto compute = [&mutex, &counter, &base_log_probs, &kld, n_vocab, logits, tokens, n_token, nv, kld_values] () {+ auto compute = [&mutex, &counter, &base_log_probs, &kld, n_vocab, logits, tokens, n_token, nv, kld_values, p_diff_values] () {kl_divergence_result local_kld;while (true) {std::unique_lock<std::mutex> lock(mutex);int i = counter++;if (i >= n_token) {- kld.sum_nll += local_kld.sum_nll;- kld.sum_nll2 += local_kld.sum_nll2;- kld.sum_kld += local_kld.sum_kld;- kld.sum_kld2 += local_kld.sum_kld2;- kld.sum_nll_diff += local_kld.sum_nll_diff;- kld.sum_nll_diff2 += local_kld.sum_nll_diff2;- kld.n_same_top += local_kld.n_same_top;- kld.count += local_kld.count;+ kld.sum_nll += local_kld.sum_nll;+ kld.sum_nll2 += local_kld.sum_nll2;+ kld.sum_nll_base += local_kld.sum_nll_base;+ kld.sum_nll_base2 += local_kld.sum_nll_base2;+ kld.sum_nll_nll_base += local_kld.sum_nll_nll_base;+ kld.sum_kld += local_kld.sum_kld;+ kld.sum_kld2 += local_kld.sum_kld2;+ kld.sum_p_diff += local_kld.sum_p_diff;+ kld.sum_p_diff2 += local_kld.sum_p_diff2;+ kld.sum_p_diff4 += local_kld.sum_p_diff4;+ kld.n_same_top += local_kld.n_same_top;+ kld.max_p_diff = std::max(kld.max_p_diff, local_kld.max_p_diff);+ kld.count += local_kld.count;break;}lock.unlock();- double v = log_softmax(n_vocab, logits + i*n_vocab, base_log_probs.data() + i*nv, tokens[i+1], local_kld);- kld_values[i] = (float)v;+ std::pair<double, float> v = log_softmax(n_vocab, logits + i*n_vocab, base_log_probs.data() + i*nv, tokens[i+1], local_kld);+ kld_values[i] = (float)v.first;+ p_diff_values[i] = v.second;}};for (auto & w : workers) {@@ -1713,7 +1739,8 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {GGML_ASSERT(llama_add_eos_token(llama_get_model(ctx)) != 1);std::vector<uint16_t> log_probs_uint16(size_t(n_ctx - 1 - n_ctx/2) * nv);- std::vector<float> kld_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);+ std::vector<float> kld_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);+ std::vector<float> p_diff_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);std::vector<float> logits;if (num_batches > 1) {logits.reserve(n_ctx * n_vocab);@@ -1730,9 +1757,18 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {df = df > 0 && count > 10 ? sqrt(df/(count-1)) : 0.;return std::make_pair(f, df);};+ auto covariance = [] (double suma, double sumb, double sumab, size_t count) {+ if (count < 10) {+ return 0.0;+ }+ double var = sumab/count - (suma/count)*(sumb/count);+ var /= count - 1;+ return var;+ };kl_divergence_result kld;- auto kld_ptr = kld_values.data();+ auto kld_ptr = kld_values.data();+ auto p_diff_ptr = p_diff_values.data();for (int i = 0; i < n_chunk; ++i) {const int start = i * n_ctx;@@ -1787,24 +1823,42 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {}fprintf(stderr, "%.2f minutes\n", total_seconds / 60.0);- printf("\nchunk PPL ln(PPL(Q)/PPL(base)) KL-Divergence Same top\n");+ printf("\nchunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p\n");}const int first = n_ctx/2;const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,- workers, log_probs_uint16, kld, kld_ptr);- kld_ptr += n_ctx - 1 - first;+ workers, log_probs_uint16, kld, kld_ptr, p_diff_ptr);+ p_diff_ptr += n_ctx - 1 - first;+ kld_ptr += n_ctx - 1 - first;++ printf("%4d", i+1);++ auto log_ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);+ const double ppl_val = exp(log_ppl.first);+ const double ppl_unc = ppl_val * log_ppl.second; // ppl_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl.second ** 2 )+ printf(" %9.4lf ± %9.4lf", ppl_val, ppl_unc);++ auto log_ppl_base = mean_and_uncertainty(kld.sum_nll_base, kld.sum_nll_base2, kld.count);+ const double log_ppl_cov = covariance(kld.sum_nll, kld.sum_nll_base, kld.sum_nll_nll_base, kld.count);+ const double log_ppl_ratio_val = log_ppl.first - log_ppl_base.first;+ const double log_ppl_ratio_unc = sqrt(log_ppl.second*log_ppl.second + log_ppl_base.second*log_ppl_base.second - 2.0*log_ppl_cov);+ printf(" %10.5lf ± %10.5lf", log_ppl_ratio_val, log_ppl_ratio_unc);++ auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);+ printf(" %10.5lf ± %10.5lf", kl_div.first, kl_div.second);- auto ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);- auto log_ppl_ratio = mean_and_uncertainty(kld.sum_nll_diff, kld.sum_nll_diff2, kld.count);- auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);- auto p_top = 1.*kld.n_same_top/kld.count;- auto d_p_top = sqrt(p_top*(1 - p_top)/(kld.count - 1));+ auto p_diff_mse = mean_and_uncertainty(kld.sum_p_diff2, kld.sum_p_diff4, kld.count);+ const double p_diff_rms_val = sqrt(p_diff_mse.first);+ const double p_diff_rms_unc = 0.5/p_diff_rms_val * p_diff_mse.second;+ printf(" %6.3lf ± %6.3lf %%", 100.0*p_diff_rms_val, 100.0*p_diff_rms_unc);- printf("%4d %10.4lf %10.5lf ± %10.5f %10.5f ± %10.5lf %.5f ± %.5f\n", i+1, exp(ppl.first),- log_ppl_ratio.first, log_ppl_ratio.second, kl_div.first, kl_div.second,- p_top, d_p_top);+ double p_top_val = 1.*kld.n_same_top/kld.count;+ double p_top_unc = sqrt(p_top_val*(1 - p_top_val)/(kld.count - 1));+ printf(" %6.3lf ± %6.3lf %%", 100.0*p_top_val, 100.0*p_top_unc);++ printf("\n");fflush(stdout);@@ -1815,31 +1869,97 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {if (kld.count < 100) return; // we do not wish to do statistics on so few valuesstd::sort(kld_values.begin(), kld_values.end());+ std::sort(p_diff_values.begin(), p_diff_values.end());++ printf("====== Perplexity statistics ======\n");++ auto log_ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);+ const double ppl_val = exp(log_ppl.first);+ const double ppl_unc = ppl_val * log_ppl.second; // ppl_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl.second ** 2 )+ printf("Mean PPL(Q) : %10.6lf ± %10.6lf\n", ppl_val, ppl_unc);++ auto log_ppl_base = mean_and_uncertainty(kld.sum_nll_base, kld.sum_nll_base2, kld.count);+ const double ppl_base_val = exp(log_ppl_base.first);+ const double ppl_base_unc = ppl_base_val * log_ppl_base.second; // ppl_base_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl_base.second ** 2 )+ printf("Mean PPL(base) : %10.6lf ± %10.6lf\n", ppl_base_val, ppl_base_unc);++ const double log_ppl_cov = covariance(kld.sum_nll, kld.sum_nll_base, kld.sum_nll_nll_base, kld.count);+ // printf("Cov(ln(PPL(Q)), ln(PPL(base))): %10.6lf\n", log_ppl_cov);+ const double log_ppl_cor = log_ppl_cov / (log_ppl.second*log_ppl_base.second);+ printf("Cor(ln(PPL(Q)), ln(PPL(base))): %6.2lf%%\n", 100.0*log_ppl_cor);++ const double log_ppl_ratio_val = log_ppl.first - log_ppl_base.first;+ const double log_ppl_ratio_unc = sqrt(log_ppl.second*log_ppl.second + log_ppl_base.second*log_ppl_base.second - 2.0*log_ppl_cov);+ printf("Mean ln(PPL(Q)/PPL(base)) : %10.6lf ± %10.6lf\n", log_ppl_ratio_val, log_ppl_ratio_unc);- printf("===== KL-divergence statistics\n");+ const double ppl_ratio_val = exp(log_ppl_ratio_val);+ const double ppl_ratio_unc = ppl_ratio_val * log_ppl_ratio_unc; // ppl_ratio_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl_ratio.second ** 2 )+ printf("Mean PPL(Q)/PPL(base) : %10.6lf ± %10.6lf\n", ppl_ratio_val, ppl_ratio_unc);++ const double ppl_cov = ppl_val * ppl_base_val * log_ppl_cov;+ const double ppl_diff_val = ppl_val - ppl_base_val;+ const double ppl_diff_unc = sqrt(ppl_unc*ppl_unc + ppl_base_unc*ppl_base_unc - 2.0*ppl_cov);+ printf("Mean PPL(Q)-PPL(base) : %10.6lf ± %10.6lf\n", ppl_diff_val, ppl_diff_unc);++ printf("\n");++ printf("====== KL divergence statistics ======\n");auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);- printf("Average: %10.6f ±%10.6lf\n", kl_div.first, kl_div.second);+ printf("Mean KLD: %10.6lf ± %10.6lf\n", kl_div.first, kl_div.second);auto kld_median = kld_values.size()%2 == 0 ? 0.5f*(kld_values[kld_values.size()/2] + kld_values[kld_values.size()/2-1]): kld_values[kld_values.size()/2];- printf("Median : %10.6f\n", kld_median);- auto percentile = [&kld_values] (float fraction) {- if (fraction <= 0) return kld_values.front();- if (fraction >= 1) return kld_values.back();- float p = fraction*(kld_values.size() - 1);+ auto percentile = [] (std::vector<float> values, float fraction) {+ if (fraction <= 0) return values.front();+ if (fraction >= 1) return values.back();+ float p = fraction*(values.size() - 1);size_t ip = size_t(p); p -= ip;- return (1 - p)*kld_values[ip] + p*kld_values[std::min(ip+1, kld_values.size()-1)];+ return (1 - p)*values[ip] + p*values[std::min(ip+1, values.size()-1)];};- printf("Maximum: %10.6f\n", kld_values.back());- printf("KLD_99 : %10.6f\n", percentile(0.99f));- printf("KLD_95 : %10.6f\n", percentile(0.95f));- printf("KLD_90 : %10.6f\n", percentile(0.90f));+ printf("Maximum KLD: %10.6f\n", kld_values.back());+ printf("99.9%% KLD: %10.6f\n", percentile(kld_values, 0.999f));+ printf("99.0%% KLD: %10.6f\n", percentile(kld_values, 0.990f));+ printf("99.0%% KLD: %10.6f\n", percentile(kld_values, 0.990f));+ printf("Median KLD: %10.6f\n", kld_median);+ printf("10.0%% KLD: %10.6f\n", percentile(kld_values, 0.100f));+ printf(" 5.0%% KLD: %10.6f\n", percentile(kld_values, 0.050f));+ printf(" 1.0%% KLD: %10.6f\n", percentile(kld_values, 0.010f));+ printf("Minimum KLD: %10.6f\n", kld_values.front());++ printf("\n");- printf("Minimum: %10.6f\n", kld_values.front());- printf("KLD_01 : %10.6f\n", percentile(0.01f));- printf("KLD_05 : %10.6f\n", percentile(0.05f));- printf("KLD_10 : %10.6f\n", percentile(0.10f));+ printf("====== Token probability statistics ======\n");++ auto p_diff = mean_and_uncertainty(kld.sum_p_diff, kld.sum_p_diff2, kld.count);+ printf("Mean Δp: %6.3lf ± %5.3lf %%\n", 100.0*p_diff.first, 100.0*p_diff.second);++ auto p_diff_median = p_diff_values.size()%2 == 0 ? 0.5f*(p_diff_values[p_diff_values.size()/2] + p_diff_values[p_diff_values.size()/2-1])+ : p_diff_values[p_diff_values.size()/2];++ printf("Maximum Δp: %6.3lf%%\n", 100.0*p_diff_values.back());+ printf("99.9%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.999f));+ printf("99.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.990f));+ printf("95.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.950f));+ printf("90.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.900f));+ printf("75.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.750f));+ printf("Median Δp: %6.3lf%%\n", 100.0*p_diff_median);+ printf("25.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.250f));+ printf("10.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.100f));+ printf(" 5.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.050f));+ printf(" 1.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.010f));+ printf(" 0.1%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.001f));+ printf("Minimum Δp: %6.3lf%%\n", 100.0*p_diff_values.front());++ auto p_diff_mse = mean_and_uncertainty(kld.sum_p_diff2, kld.sum_p_diff4, kld.count);+ // printf("MSE Δp : %10.6lf ± %10.6lf\n", p_diff_mse.first, p_diff_mse.second);++ const double p_diff_rms_val = sqrt(p_diff_mse.first);+ const double p_diff_rms_unc = 0.5/p_diff_rms_val * p_diff_mse.second;+ printf("RMS Δp : %6.3lf ± %5.3lf %%\n", 100.0*p_diff_rms_val, 100.0*p_diff_rms_unc);++ const double same_top_p = 1.0*kld.n_same_top/kld.count;+ printf("Same top p: %6.3lf ± %5.3lf %%\n", 100.0*same_top_p, 100.0*sqrt(same_top_p*(1.0 - same_top_p)/(kld.count - 1)));}diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex d72b2719..27a9636e 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -59,6 +59,11 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {{ "COPY", LLAMA_FTYPE_ALL_F32, "only copy tensors, no quantizing", },};+static const char * const LLM_KV_QUANTIZE_IMATRIX_FILE = "quantize.imatrix.file";+static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrix.dataset";+static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";+static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";+static bool is_integer_str(const char *s) {if (*s == '-') ++s;if (!*s) return false;@@ -122,7 +127,7 @@ static void usage(const char * executable) {exit(1);}-static void load_imatrix(const std::string & imatrix_file, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {+static int load_imatrix(const std::string & imatrix_file, std::string & imatrix_dataset, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {std::ifstream in(imatrix_file.c_str(), std::ios::binary);if (!in) {printf("%s: failed to open %s\n",__func__, imatrix_file.c_str());@@ -169,18 +174,33 @@ static void load_imatrix(const std::string & imatrix_file, std::unordered_map<stprintf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n", __func__, int(e.size()), ncall, name.c_str());}}- printf("%s: loaded %d importance matrix entries from %s\n", __func__, int(imatrix_data.size()), imatrix_file.c_str());++ // latest imatrix version contains the dataset filename at the end of the file+ int m_last_call = 0;+ if (in.peek() != EOF) {+ in.read((char *)&m_last_call, sizeof(m_last_call));+ int dataset_len;+ in.read((char *)&dataset_len, sizeof(dataset_len));+ std::vector<char> dataset_as_vec(dataset_len);+ in.read(dataset_as_vec.data(), dataset_len);+ imatrix_dataset.assign(dataset_as_vec.begin(), dataset_as_vec.end());+ printf("%s: imatrix dataset='%s'\n", __func__, imatrix_dataset.c_str());+ }+ printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_call);+ return m_last_call;}-static void prepare_imatrix(const std::string & imatrix_file,+static int prepare_imatrix(const std::string & imatrix_file,+ std::string & imatrix_dataset,const std::vector<std::string> & included_weights,const std::vector<std::string> & excluded_weights,std::unordered_map<std::string, std::vector<float>> & imatrix_data) {+ int m_last_call = -1;if (!imatrix_file.empty()) {- load_imatrix(imatrix_file, imatrix_data);+ m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data);}if (imatrix_data.empty()) {- return;+ return m_last_call;}if (!excluded_weights.empty()) {for (auto& name : excluded_weights) {@@ -206,6 +226,7 @@ static void prepare_imatrix(const std::string & imatrix_file,if (!imatrix_data.empty()) {printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size()));}+ return m_last_call;}static ggml_type parse_ggml_type(const char * arg) {@@ -220,43 +241,6 @@ static ggml_type parse_ggml_type(const char * arg) {return result;}-static bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {- const char* sep = strchr(data, '=');- if (sep == nullptr || sep - data >= 128) {- fprintf(stderr, "%s: malformed KV override '%s'\n", __func__, data);- return false;- }- llama_model_kv_override kvo;- std::strncpy(kvo.key, data, sep - data);- kvo.key[sep - data] = 0;- sep++;- if (strncmp(sep, "int:", 4) == 0) {- sep += 4;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;- kvo.int_value = std::atol(sep);- } else if (strncmp(sep, "float:", 6) == 0) {- sep += 6;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;- kvo.float_value = std::atof(sep);- } else if (strncmp(sep, "bool:", 5) == 0) {- sep += 5;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;- if (std::strcmp(sep, "true") == 0) {- kvo.bool_value = true;- } else if (std::strcmp(sep, "false") == 0) {- kvo.bool_value = false;- } else {- fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data);- return false;- }- } else {- fprintf(stderr, "%s: invalid type for KV override '%s'\n", __func__, data);- return false;- }- overrides.emplace_back(std::move(kvo));- return true;-}-int main(int argc, char ** argv) {if (llamafile_has(argv, "--version")) {@@ -340,10 +324,43 @@ int main(int argc, char ** argv) {usage(argv[0]);}+ std::string imatrix_dataset;std::unordered_map<std::string, std::vector<float>> imatrix_data;- prepare_imatrix(imatrix_file, included_weights, excluded_weights, imatrix_data);+ int m_last_call = prepare_imatrix(imatrix_file, imatrix_dataset, included_weights, excluded_weights, imatrix_data);if (!imatrix_data.empty()) {params.imatrix = &imatrix_data;+ {+ llama_model_kv_override kvo;+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_FILE);+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;+ strncpy(kvo.val_str, imatrix_file.c_str(), 127);+ kvo.val_str[127] = '\0';+ kv_overrides.emplace_back(std::move(kvo));+ }+ if (!imatrix_dataset.empty()) {+ llama_model_kv_override kvo;+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_DATASET);+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;+ strncpy(kvo.val_str, imatrix_dataset.c_str(), 127);+ kvo.val_str[127] = '\0';+ kv_overrides.emplace_back(std::move(kvo));+ }++ {+ llama_model_kv_override kvo;+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES);+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;+ kvo.val_i64 = imatrix_data.size();+ kv_overrides.emplace_back(std::move(kvo));+ }++ if (m_last_call > 0) {+ llama_model_kv_override kvo;+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS);+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;+ kvo.val_i64 = m_last_call;+ kv_overrides.emplace_back(std::move(kvo));+ }}if (!kv_overrides.empty()) {kv_overrides.emplace_back();diff --git a/llama.cpp/quantize_row_q8_0.inc b/llama.cpp/quantize_row_q8_0.incdeleted file mode 100644index f4fe9d95..00000000--- a/llama.cpp/quantize_row_q8_0.inc+++ /dev/null@@ -1,187 +0,0 @@-static void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) {- assert(QK8_0 == 32);- assert(k % QK8_0 == 0);- const int nb = k / QK8_0;-- block_q8_0 * restrict y = vy;--#if defined(__ARM_NEON)- for (int i = 0; i < nb; i++) {- float32x4_t srcv [8];- float32x4_t asrcv[8];- float32x4_t amaxv[8];-- for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);- for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);-- for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);- for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);- for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);-- const float amax = vmaxvq_f32(amaxv[0]);-- const float d = amax / ((1 << 7) - 1);- const float id = d ? 1.0f/d : 0.0f;-- y[i].d = GGML_FP32_TO_FP16(d);-- for (int j = 0; j < 8; j++) {- const float32x4_t v = vmulq_n_f32(srcv[j], id);- const int32x4_t vi = vcvtnq_s32_f32(v);-- y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);- y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);- y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);- y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);- }- }-#elif defined(__wasm_simd128__)- for (int i = 0; i < nb; i++) {- v128_t srcv [8];- v128_t asrcv[8];- v128_t amaxv[8];-- for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);- for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);-- for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);- for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);- for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);-- const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),- wasm_f32x4_extract_lane(amaxv[0], 1)),- MAX(wasm_f32x4_extract_lane(amaxv[0], 2),- wasm_f32x4_extract_lane(amaxv[0], 3)));-- const float d = amax / ((1 << 7) - 1);- const float id = d ? 1.0f/d : 0.0f;-- y[i].d = GGML_FP32_TO_FP16(d);-- for (int j = 0; j < 8; j++) {- const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));- const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);-- y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);- y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);- y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);- y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);- }- }-#elif defined(__AVX2__) || defined(__AVX__)- for (int i = 0; i < nb; i++) {- // Load elements into 4 AVX vectors- __m256 v0 = _mm256_loadu_ps( x );- __m256 v1 = _mm256_loadu_ps( x + 8 );- __m256 v2 = _mm256_loadu_ps( x + 16 );- __m256 v3 = _mm256_loadu_ps( x + 24 );- x += 32;-- // Compute max(abs(e)) for the block- const __m256 signBit = _mm256_set1_ps( -0.0f );- __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );- maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );- maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );- maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );-- __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );- max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );- max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );- const float maxScalar = _mm_cvtss_f32( max4 );-- // Quantize these floats- const float d = maxScalar / 127.f;- y[i].d = GGML_FP32_TO_FP16(d);- const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;- const __m256 mul = _mm256_set1_ps( id );-- // Apply the multiplier- v0 = _mm256_mul_ps( v0, mul );- v1 = _mm256_mul_ps( v1, mul );- v2 = _mm256_mul_ps( v2, mul );- v3 = _mm256_mul_ps( v3, mul );-- // Round to nearest integer- v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );- v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );- v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );- v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );-- // Convert floats to integers- __m256i i0 = _mm256_cvtps_epi32( v0 );- __m256i i1 = _mm256_cvtps_epi32( v1 );- __m256i i2 = _mm256_cvtps_epi32( v2 );- __m256i i3 = _mm256_cvtps_epi32( v3 );--#if defined(__AVX2__)- // Convert int32 to int16- i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15- i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31- // Convert int16 to int8- i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31-- // We got our precious signed bytes, but the order is now wrong- // These AVX2 pack instructions process 16-byte pieces independently- // The following instruction is fixing the order- const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );- i0 = _mm256_permutevar8x32_epi32( i0, perm );-- _mm256_storeu_si256((__m256i *)y[i].qs, i0);-#else- // Since we don't have in AVX some necessary functions,- // we split the registers in half and call AVX2 analogs from SSE- __m128i ni0 = _mm256_castsi256_si128( i0 );- __m128i ni1 = _mm256_extractf128_si256( i0, 1);- __m128i ni2 = _mm256_castsi256_si128( i1 );- __m128i ni3 = _mm256_extractf128_si256( i1, 1);- __m128i ni4 = _mm256_castsi256_si128( i2 );- __m128i ni5 = _mm256_extractf128_si256( i2, 1);- __m128i ni6 = _mm256_castsi256_si128( i3 );- __m128i ni7 = _mm256_extractf128_si256( i3, 1);-- // Convert int32 to int16- ni0 = _mm_packs_epi32( ni0, ni1 );- ni2 = _mm_packs_epi32( ni2, ni3 );- ni4 = _mm_packs_epi32( ni4, ni5 );- ni6 = _mm_packs_epi32( ni6, ni7 );- // Convert int16 to int8- ni0 = _mm_packs_epi16( ni0, ni2 );- ni4 = _mm_packs_epi16( ni4, ni6 );-- _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);- _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);-#endif- }-#elif defined(__riscv_v_intrinsic)-- size_t vl = __riscv_vsetvl_e32m4(QK8_0);-- for (int i = 0; i < nb; i++) {- // load elements- vfloat32m4_t v_x = __riscv_vle32_v_f32m4(x+i*QK8_0, vl);-- vfloat32m4_t vfabs = __riscv_vfabs_v_f32m4(v_x, vl);- vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl);- vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m4_f32m1(vfabs, tmp, vl);- float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);-- const float d = amax / ((1 << 7) - 1);- const float id = d ? 1.0f/d : 0.0f;-- y[i].d = GGML_FP32_TO_FP16(d);-- vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);-- // convert to integer- vint16m2_t vi = __riscv_vfncvt_x_f_w_i16m2(x0, vl);- vint8m1_t vs = __riscv_vncvt_x_x_w_i8m1(vi, vl);-- // store result- __riscv_vse8_v_i8m1(y[i].qs , vs, vl);- }-#else- GGML_UNUSED(nb);- // scalar- quantize_row_q8_0_reference(x, y, k);-#endif-}diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cppindex 447a6864..f42f5b92 100644--- a/llama.cpp/sampling.cpp+++ b/llama.cpp/sampling.cpp@@ -71,7 +71,7 @@ void llama_sampling_reset(llama_sampling_context * ctx) {void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed) {if (seed == LLAMA_DEFAULT_SEED) {- seed = time(NULL);+ seed = std::random_device{}();}ctx->rng.seed(seed);}diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 441351d8..2d869576 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -2671,52 +2671,17 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,break;}sparams.chat_template = argv[i];- }- else if (arg == "--override-kv")- {+ } else if (arg == "--override-kv") {if (++i >= argc) {invalid_param = true;break;}- char * sep = strchr(argv[i], '=');- if (sep == nullptr || sep - argv[i] >= 128) {- fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);- invalid_param = true;- break;- }- struct llama_model_kv_override kvo;- std::strncpy(kvo.key, argv[i], sep - argv[i]);- kvo.key[sep - argv[i]] = 0;- sep++;- if (strncmp(sep, "int:", 4) == 0) {- sep += 4;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;- kvo.int_value = std::atol(sep);- } else if (strncmp(sep, "float:", 6) == 0) {- sep += 6;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;- kvo.float_value = std::atof(sep);- } else if (strncmp(sep, "bool:", 5) == 0) {- sep += 5;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;- if (std::strcmp(sep, "true") == 0) {- kvo.bool_value = true;- } else if (std::strcmp(sep, "false") == 0) {- kvo.bool_value = false;- } else {- fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);- invalid_param = true;- break;- }- } else {+ if (!parse_kv_override(argv[i], params.kv_overrides)) {fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);invalid_param = true;break;}- params.kv_overrides.push_back(kvo);- }- else- {+ } else {fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());exit(1);}diff --git a/llama.cpp/unicode-data.cpp b/llama.cpp/unicode-data.cppindex 22f8b0f0..e6bafb3a 100644--- a/llama.cpp/unicode-data.cpp+++ b/llama.cpp/unicode-data.cpp@@ -1,4 +1,4 @@-#include "unicode-data.h"+#include "unicode-data.h"#include <cstdint>#include <map>diff --git a/llama.cpp/unicode-data.h b/llama.cpp/unicode-data.hindex b99500b8..cb9dd8aa 100644--- a/llama.cpp/unicode-data.h+++ b/llama.cpp/unicode-data.h@@ -12,5 +12,5 @@ extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_maextern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation;extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;-extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;-extern const std::map<char32_t, char32_t> unicode_map_lowercase;+extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;+extern const std::map<char32_t, char32_t> unicode_map_lowercase;diff --git a/llama.cpp/unicode.cpp b/llama.cpp/unicode.cppindex df8c5f58..f2ccda05 100644--- a/llama.cpp/unicode.cpp+++ b/llama.cpp/unicode.cpp@@ -5,11 +5,14 @@#include <cstddef>#include <cstdint>#include <map>+#include <regex>#include <stdexcept>#include <string>#include <unordered_map>#include <utility>#include <vector>+#include <locale>+#include <codecvt>static std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {std::string result;@@ -53,23 +56,22 @@ static uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset)offset += 4;return result;}- throw std::invalid_argument("invalid string");+ throw std::invalid_argument("failed to convert utf8 to codepoint");}-static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {- std::vector<uint16_t> result;- if (/* 0x0000 <= cp && */ cp <= 0xffff) {- result.emplace_back(cp);- }- else if (0x10000 <= cp && cp <= 0x10ffff) {- result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));- result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));- }- else {- throw std::invalid_argument("invalid cpt");- }- return result;-}+//static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {+// std::vector<uint16_t> result;+// if (/* 0x0000 <= cp && */ cp <= 0xffff) {+// result.emplace_back(cp);+// return result;+// }+// if (0x10000 <= cp && cp <= 0x10ffff) {+// result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));+// result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));+// return result;+// }+// throw std::invalid_argument("failed to convert codepoint to utf16");+//}//static std::vector<uint16_t> unicode_cpts_to_utf16(const std::vector<uint32_t> & cps) {// std::vector<uint16_t> result;@@ -80,28 +82,28 @@ static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {// return result;//}-static uint32_t cpt_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {- assert(offset < utf16.size());- if (((utf16[0] >> 10) << 10) != 0xd800) {- auto result = utf16[offset + 0];- offset += 1;- return result;- }-- if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) {- throw std::invalid_argument("invalid character");- }-- auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));- offset += 2;- return result;-}+//static uint32_t unicode_cpt_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {+// assert(offset < utf16.size());+// if (((utf16[0] >> 10) << 10) != 0xd800) {+// auto result = utf16[offset + 0];+// offset += 1;+// return result;+// }+//+// if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) {+// throw std::invalid_argument("invalid character");+// }+//+// auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));+// offset += 2;+// return result;+//}//static std::vector<uint32_t> unicode_cpts_from_utf16(const std::vector<uint16_t> & utf16) {// std::vector<uint32_t> result;// size_t offset = 0;// while (offset < utf16.size()) {-// result.push_back(cpt_from_utf16(utf16, offset));+// result.push_back(unicode_cpt_from_utf16(utf16, offset));// }// return result;//}@@ -194,34 +196,277 @@ static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {return map;}+static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {+ std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;+ return conv.from_bytes(s);+}++static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {+ std::vector<std::string> bpe_encoded_words;+ for (const auto & word : bpe_words) {+ std::string text_utf;+ auto utf_word = unicode_cpts_from_utf8(word);+ for (size_t i = 0; i < utf_word.size(); ++i) {+ text_utf += unicode_cpt_to_utf8(utf_word[i]);+ }++ std::string encoded_token;+ for (char & c : text_utf) {+ encoded_token += unicode_byte_to_utf8(c);+ }+ bpe_encoded_words.emplace_back(encoded_token);+ }+ return bpe_encoded_words;+}++// GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s++static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) {+ std::vector<size_t> bpe_offsets; // store the offset of each word+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size++ size_t start = 0;++ const auto cpts = unicode_cpts_from_utf8(text);++ for (auto offset : offsets) {+ std::string token;++ bool collecting_numeric = false;+ bool collecting_letter = false;+ bool collecting_special = false;+ bool collecting_whitespace_lookahead = false;+ bool collecting = false;++ std::vector<std::string> text_utf;+ text_utf.reserve(offset);++ for (size_t i = start; i < start + offset; ++i) {+ text_utf.emplace_back(unicode_cpt_to_utf8(cpts[i]));+ }++ for (int i = 0; i < (int)text_utf.size(); i++) {+ const std::string & utf_char = text_utf[i];+ bool split_condition = false;+ int bytes_remain = text_utf.size() - i;++ // forward backward lookups+ const std::string & utf_char_next = (i + 1 < (int)text_utf.size()) ? text_utf[i + 1] : "";+ const std::string & utf_char_next_next = (i + 2 < (int)text_utf.size()) ? text_utf[i + 2] : "";++ // handling contractions+ if (!split_condition && bytes_remain >= 2) {+ // 's|'t|'m|'d+ if (utf_char == "\'" && (utf_char_next == "s" || utf_char_next == "t" || utf_char_next == "m" || utf_char_next == "d")) {+ split_condition = true;+ }+ if (split_condition) {+ if (token.size()) {+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());+ }+ token = utf_char + utf_char_next;+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());+ token = "";+ i++;+ continue;+ }+ }+ if (!split_condition && bytes_remain >= 3) {+ // 're|'ve|'ll+ if (utf_char == "\'" && (+ (utf_char_next == "r" && utf_char_next_next == "e") ||+ (utf_char_next == "v" && utf_char_next_next == "e") ||+ (utf_char_next == "l" && utf_char_next_next == "l"))+ ) {+ split_condition = true;+ }+ if (split_condition) {+ // current token + next token can be defined+ if (token.size()) {+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());+ }+ token = utf_char;+ token += utf_char_next;+ token += utf_char_next_next;++ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());+ token = "";+ i += 2;+ continue;+ }+ }++ if (!split_condition && !collecting) {+ if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) {+ collecting_letter = true;+ collecting = true;+ }+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {+ collecting_numeric = true;+ collecting = true;+ }+ else if (+ ((unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (unicode_cpt_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) ||+ (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE)+ ) {+ collecting_special = true;+ collecting = true;+ }+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) {+ collecting_whitespace_lookahead = true;+ collecting = true;+ }+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) {+ split_condition = true;+ }+ }+ else if (!split_condition && collecting) {+ if (collecting_letter && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER) {+ split_condition = true;+ }+ else if (collecting_numeric && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) {+ split_condition = true;+ }+ else if (collecting_special && (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) {+ split_condition = true;+ }+ else if (collecting_whitespace_lookahead && (unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {+ split_condition = true;+ }+ }++ if (utf_char_next == "") {+ split_condition = true; // final+ token += utf_char;+ }++ if (split_condition) {+ if (token.size()) {+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());+ }+ token = utf_char;+ collecting = false;+ collecting_letter = false;+ collecting_numeric = false;+ collecting_special = false;+ collecting_whitespace_lookahead = false;+ }+ else {+ token += utf_char;+ }+ }++ start += offset;+ }++ return bpe_offsets;+}++// use std::wregex to split the text+static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, const std::wstring & regex_expr, const std::vector<size_t> & offsets) {+ std::wregex expr(regex_expr);+ std::vector<size_t> bpe_offsets; // store the offset of each word+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size+ size_t start = 0;+ for (auto offset : offsets) {+ std::wcregex_iterator it(wtext.data() + start, wtext.data() + start + offset, expr);+ std::wcregex_iterator end;++ int64_t start_idx = 0;+ while (it != end) {+ std::wcmatch match = *it;+ if (match.position() > start_idx) {+ bpe_offsets.emplace_back(match.position() - start_idx);+ }+ bpe_offsets.emplace_back(match.length());+ start_idx = match.position() + match.length();+ ++it;+ }++ if (start_idx < (int64_t) offset) {+ bpe_offsets.emplace_back(offset - start_idx);+ }+ start += offset;+ }++ return bpe_offsets;+}++// use std::regex to split the text+static std::vector<size_t> unicode_regex_split_stl(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {+ std::regex expr(regex_expr);+ std::vector<size_t> bpe_offsets; // store the offset of each word+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size+ size_t start = 0;+ for (auto offset : offsets) {+ std::cregex_iterator it(text.data() + start, text.data() + start + offset, expr);+ std::cregex_iterator end;++ int64_t start_idx = 0;+ while (it != end) {+ std::cmatch match = *it;+ if (match.position() > start_idx) {+ bpe_offsets.emplace_back(match.position() - start_idx);+ }+ bpe_offsets.emplace_back(match.length());+ start_idx = match.position() + match.length();+ ++it;+ }++ if (start_idx < (int64_t) offset) {+ bpe_offsets.emplace_back(offset - start_idx);+ }+ start += offset;+ }++ return bpe_offsets;+}++static std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {+ std::vector<size_t> bpe_offsets;++ (void)(text);+ (void)(regex_expr);+ (void)(offsets);+ // TODO: this implementation is actually wrong, uncomment and run:+ // make -j && ./bin/test-tokenizer-0 ../models/ggml-vocab-gpt-2.gguf+ //if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") {+ // bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);+ //}++ return bpe_offsets;+}+//// interface//std::string unicode_cpt_to_utf8(uint32_t cp) {std::string result;+if (/* 0x00 <= cp && */ cp <= 0x7f) {result.push_back(cp);+ return result;}- else if (0x80 <= cp && cp <= 0x7ff) {+ if (0x80 <= cp && cp <= 0x7ff) {result.push_back(0xc0 | ((cp >> 6) & 0x1f));result.push_back(0x80 | (cp & 0x3f));+ return result;}- else if (0x800 <= cp && cp <= 0xffff) {+ if (0x800 <= cp && cp <= 0xffff) {result.push_back(0xe0 | ((cp >> 12) & 0x0f));result.push_back(0x80 | ((cp >> 6) & 0x3f));result.push_back(0x80 | (cp & 0x3f));+ return result;}- else if (0x10000 <= cp && cp <= 0x10ffff) {+ if (0x10000 <= cp && cp <= 0x10ffff) {result.push_back(0xf0 | ((cp >> 18) & 0x07));result.push_back(0x80 | ((cp >> 12) & 0x3f));result.push_back(0x80 | ((cp >> 6) & 0x3f));result.push_back(0x80 | (cp & 0x3f));+ return result;}- else {- throw std::invalid_argument("invalid codepoint");- }- return result;++ throw std::invalid_argument("invalid codepoint");}std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {@@ -275,3 +520,167 @@ char32_t unicode_tolower(char32_t cp) {auto it = unicode_map_lowercase.find(cp);return it == unicode_map_lowercase.end() ? cp : it->second;}++std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {+ // unicode categories+ static const std::map<std::string, int> k_ucat_enum = {+ { "\\p{N}", CODEPOINT_TYPE_DIGIT },+ { "\\p{L}", CODEPOINT_TYPE_LETTER },+ { "\\p{P}", CODEPOINT_TYPE_PUNCTUATION },+ };++ static const std::map<int, int> k_ucat_cpt = {+ { CODEPOINT_TYPE_DIGIT, 0xD1 },+ { CODEPOINT_TYPE_LETTER, 0xD2 },+ { CODEPOINT_TYPE_PUNCTUATION, 0xD3 },+ };++ static const std::map<int, std::string> k_ucat_map = {+ { CODEPOINT_TYPE_DIGIT, "\x30-\x39" }, // 0-9+ { CODEPOINT_TYPE_LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z+ { CODEPOINT_TYPE_PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}+ };++ // compute collapsed codepoints only if needed by at least one regex+ bool need_collapse = false;+ for (auto & regex_expr : regex_exprs) {+ // search for unicode categories+ for (const auto & ucat : k_ucat_enum) {+ if (std::string::npos != regex_expr.find(ucat.first)) {+ need_collapse = true;+ break;+ }+ }+ }++ const auto cpts = unicode_cpts_from_utf8(text);++ // generate a "collapsed" representation of the text, where all codepoints are replaced by a single byte+ // ref: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2081479935+ std::string text_collapsed;+ if (need_collapse) {+ // collapse all unicode categories+ text_collapsed.resize(cpts.size());++ for (size_t i = 0; i < cpts.size(); ++i) {+ // keep single-byte codepoints as is+ if (cpts[i] < 128) {+ text_collapsed[i] = cpts[i];+ continue;+ }++ const int cpt_type = unicode_cpt_type(cpts[i]);++ if (k_ucat_cpt.find(cpt_type) != k_ucat_cpt.end()) {+ text_collapsed[i] = k_ucat_cpt.at(cpt_type);+ } else {+ text_collapsed[i] = (char) 0xD0; // fallback+ }+ }+ }++ std::vector<size_t> bpe_offsets = { cpts.size() };++ for (auto & regex_expr : regex_exprs) {+ // first, see if we have an efficient custom regex implementation+ auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets);++ if (!tmp.empty()) {+ bpe_offsets = std::move(tmp);+ continue;+ }++ // fallback to general-purpose std::regex / std::wregex+ try {+ // if a unicode category is used in the regex, we use the collapsed text and replace the unicode category+ // with the corresponding collapsed representation+ bool use_collapsed = false;+ for (auto & ucat : k_ucat_enum) {+ if (std::string::npos != regex_expr.find(ucat.first)) {+ use_collapsed = true;+ break;+ }+ }++ if (use_collapsed) {+ // sanity-check that the original regex does not contain any non-ASCII characters+ const auto cpts_regex = unicode_cpts_from_utf8(regex_expr);+ for (size_t i = 0; i < cpts_regex.size(); ++i) {+ if (cpts_regex[i] >= 128) {+ throw std::runtime_error("Regex includes both unicode categories and non-ASCII characters - not supported");+ }+ }++ // generate a collapsed representation of the regex+ std::string regex_expr_collapsed;++ // track if we are inside [], because nested [] are not allowed+ bool inside = false;+ for (size_t i = 0; i < regex_expr.size(); ++i) {+ if (regex_expr[i] == '[' && (i == 0 || regex_expr[i - 1] != '\\')) {+ regex_expr_collapsed += '[';+ inside = true;+ continue;+ }++ if (inside && regex_expr[i] == ']' && regex_expr[i - 1] != '\\') {+ regex_expr_collapsed += ']';+ inside = false;+ continue;+ }++ if (regex_expr[i + 0] == '\\' && i + 4 < regex_expr.size() &&+ regex_expr[i + 1] == 'p' &&+ regex_expr[i + 2] == '{' &&+ regex_expr[i + 4] == '}') {+ const std::string pat = regex_expr.substr(i, 5);+ if (k_ucat_enum.find(pat) != k_ucat_enum.end()) {+ if (!inside) {+ regex_expr_collapsed += '[';+ }+ regex_expr_collapsed += k_ucat_cpt.at(k_ucat_enum.at(pat));+ regex_expr_collapsed += k_ucat_map.at(k_ucat_enum.at(pat));+ if (!inside) {+ regex_expr_collapsed += ']';+ }+ i += 4;+ continue;+ }+ }++ regex_expr_collapsed += regex_expr[i];+ }++ //printf("text_collapsed: %s\n", text_collapsed.c_str());+ //printf("regex_expr_collapsed: %s\n", regex_expr_collapsed.c_str());+ bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets);+ } else {+ // no unicode category used, we can use std::wregex directly+ const std::wstring wtext = unicode_wstring_from_utf8(text);+ const std::wstring wregex_expr = unicode_wstring_from_utf8(regex_expr);++ //printf("text: %s\n", text.c_str());+ //printf("regex_expr: %s\n", regex_expr.c_str());+ bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets);+ }+ } catch (std::regex_error & e) {+ fprintf(stderr, "Failed to process regex: '%s'\n", regex_expr.c_str());+ fprintf(stderr, "Regex error: %s\n", e.what());+ throw std::runtime_error("Failed to process regex");+ }+ }++ std::vector<std::string> bpe_words;+ bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size++ size_t start = 0;+ for (size_t & offset : bpe_offsets) {+ bpe_words.emplace_back();+ for (size_t i = start; i < start + offset; ++i) {+ bpe_words.back() += unicode_cpt_to_utf8(cpts[i]);+ }+ start += offset;+ }++ return unicode_byte_encoding_process(bpe_words);+}diff --git a/llama.cpp/unicode.h b/llama.cpp/unicode.hindex 6a0be393..ce2bcef5 100644--- a/llama.cpp/unicode.h+++ b/llama.cpp/unicode.h@@ -24,5 +24,6 @@ int unicode_cpt_type(const std::string & utf8);std::string unicode_byte_to_utf8(uint8_t byte);uint8_t unicode_utf8_to_byte(const std::string & utf8);-// simple tolower that only implements one-to-one mapping, not one-to-manychar32_t unicode_tolower(char32_t cp);++std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex f0e64ec4..352f2ee4 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -89,6 +89,7 @@ GGML_CALL int ggml_backend_cuda_reg_devices(void);static struct Cuda {bool supported;+ bool has_amd_gpu;atomic_uint once;typeof(ggml_cuda_link) *GGML_CALL link;typeof(ggml_backend_cuda_buffer_type) *GGML_CALL buffer_type;@@ -217,11 +218,12 @@ struct StringList {size_t length;};-static int StringCompare(const void* a, const void* b) {- return strcmp(*(const char**)a, *(const char**)b);+static int StringCompare(const void *a, const void *b) {+ return strcmp(*(const char **)a, *(const char **)b);}-static void AddStringToStringList(struct StringList *string_list, char *string, size_t string_length) {+static void AddStringToStringList(struct StringList *string_list, char *string,+ size_t string_length) {struct StringListEntry *new_entry = malloc(sizeof(struct StringListEntry));new_entry->string = malloc(sizeof(char) * (string_length + 1));strncpy(new_entry->string, string, string_length);@@ -376,7 +378,7 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {char **names = NULL;CopyStringListToStringArray(&name_list, &names);- qsort(names, name_list.length, sizeof(char*), StringCompare);+ qsort(names, name_list.length, sizeof(char *), StringCompare);int num_names = RemoveDuplicatesFromSortedStringArray(names, name_list.length);@@ -808,6 +810,7 @@ static bool import_cuda_impl(void) {if (FLAG_nocompile) {if ((FileExists(dso) || extract_cuda_dso(dso, "ggml-rocm")) &&link_cuda_dso(dso, library_path)) {+ ggml_cuda.has_amd_gpu = true;return true;} else {goto TryNvidia;@@ -820,10 +823,12 @@ static bool import_cuda_impl(void) {case -1:return false;case false:- if (link_cuda_dso(dso, library_path))+ if (link_cuda_dso(dso, library_path)) {+ ggml_cuda.has_amd_gpu = true;return true;- else+ } else {goto TryNvidia;+ }case true:break;default:@@ -834,10 +839,12 @@ static bool import_cuda_impl(void) {// Try building CUDA with ROCm SDK.if (compiler_path) {if (compile_amd(compiler_path, dso, src)) {- if (link_cuda_dso(dso, library_path))+ if (link_cuda_dso(dso, library_path)) {+ ggml_cuda.has_amd_gpu = true;return true;- else+ } else {goto TryNvidia;+ }}} else {tinylog(__func__,@@ -848,10 +855,12 @@ static bool import_cuda_impl(void) {// Try extracting prebuilt tinyBLAS DSO from PKZIP.if (extract_cuda_dso(dso, "ggml-rocm")) {- if (link_cuda_dso(dso, library_path))+ if (link_cuda_dso(dso, library_path)) {+ ggml_cuda.has_amd_gpu = true;return true;- else+ } else {goto TryNvidia;+ }}break;@@ -934,6 +943,11 @@ bool llamafile_has_cuda(void) {return ggml_cuda.supported;}+bool llamafile_has_amd_gpu(void) {+ cosmo_once(&ggml_cuda.once, import_cuda);+ return ggml_cuda.has_amd_gpu;+}+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {if (!llamafile_has_cuda())return 0;diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 514b7151..0503676c 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -50,11 +50,12 @@ bool llamafile_has_gpu(void);int llamafile_gpu_layers(int);bool llamafile_has_cuda(void);bool llamafile_has_metal(void);+bool llamafile_has_amd_gpu(void);int llamafile_gpu_parse(const char *);const char *llamafile_describe_gpu(void);bool llamafile_sgemm(long, long, long, const void *, long, const void *, long, void *, long, int,- int, int, int, int, int);+ int, int, int, int, int, int);struct ggml_tensor;struct ggml_compute_params;diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 6d52ebb0..172bd41e 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -111,12 +111,14 @@ static const struct GemmFuncs {* @param Atype is GGML data type of `A`* @param Btype is GGML data type of `B`* @param Ctype is GGML data type of `C`+ * @param precision may be used to control the internal compute type* @return true if this function was able to service the matmul request*/bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,- int Ctype) {- return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype);+ void *C, long ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype,+ int precision) {+ return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype,+ precision);}/**diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex f734e9b3..9201cef6 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -8,23 +8,23 @@ struct ggml_tensor;struct ggml_compute_params;bool llamafile_sgemm_unsupported(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int, int, int);bool llamafile_sgemm_amd_avx(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int, int, int);bool llamafile_sgemm_amd_fma(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int, int, int);bool llamafile_sgemm_amd_avx2(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int, int, int);bool llamafile_sgemm_amd_avxvnni(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int, int, int);bool llamafile_sgemm_amd_avx512f(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int, int, int);bool llamafile_sgemm_amd_zen4(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int, int, int);bool llamafile_sgemm_arm80(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int, int, int);bool llamafile_sgemm_arm82(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int, int, int);bool llamafile_mixmul_unsupported(struct ggml_compute_params *, const struct ggml_tensor *,const struct ggml_tensor *, const struct ggml_tensor *,diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cppindex 3825989c..b7e49462 100644--- a/llamafile/sgemm_sss_test.cpp+++ b/llamafile/sgemm_sss_test.cpp@@ -47,7 +47,7 @@ int main(int argc, char *argv[]) {BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));for (int i = 0; i < m; ++i)for (int j = 0; j < n; ++j) {diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cppindex 283973ad..651e230c 100644--- a/llamafile/sgemm_vecdot_test.cpp+++ b/llamafile/sgemm_vecdot_test.cpp@@ -46,7 +46,7 @@ int main(int argc, char *argv[]) {BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));double err_sum = 0;long long err_worst = 0;diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.incindex 7e751699..1e29bada 100644--- a/llamafile/tinyblas_cpu.inc+++ b/llamafile/tinyblas_cpu.inc@@ -383,12 +383,22 @@ inline __m512bh load(const float *p) {#endif // __AVX512BF16__////////////////////////////////////////////////////////////////////////////////////////////////////-// FLOATING POINT MATRIX MULTIPLICATION+// FLOATING POINT OUTPUT STREAMING++inline void store(float *p, float f) {+ *p = f;+}++inline void store(ggml_fp16_t *p, float f) {+ *p = GGML_FP32_TO_FP16(f);+}-float id(float x) {- return x;+inline void store(ggml_bf16_t *p, float f) {+ *p = GGML_FP32_TO_BF16(f);}-float (*veil)(float) = id;++////////////////////////////////////////////////////////////////////////////////////////////////////+// FLOATING POINT MATRIX MULTIPLICATIONtemplate <int CONFIG, int KN, typename D, typename V, typename TA, typename TB, typename TC>class tinyBLAS {@@ -579,7 +589,7 @@ class tinyBLAS {Cv[j][i]);for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]); // + logf(veil(-1));+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));}}@@ -700,7 +710,7 @@ class tinyBLAS_Q0_ARM {}for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));}}@@ -888,7 +898,7 @@ class tinyBLAS_Q0_AVX2 {}for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));}}@@ -926,52 +936,10 @@ class tinyBLAS_Q0_AVX2 {};#endif // __AVX2__-} // namespace--/**- * Performs optimized matrix multiplication on CPU.- *- * This subroutine may compute C = Aᵀ * B with column major ordering.- * Despite its name, this isn't a generalized implementation. Work is- * only performed when a handwritten kernel is written and available.- * Otherwise the caller should fall back to a general matmul routine.- *- * For example, for single-threaded single-precision GEMM you can say- *- * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1,- * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);- *- * @param m is rows in `A` and `C`- * @param n is cols in `B` and `C`- * @param k is cols in `A` and rows in `B`- * @param A is first input matrix (always transposed)- * @param lda is row stride of `A`- * @param B is second input matrix (never transposed)- * @param ldb is row stride of `B`- * @param C is input/output array of output matrices- * @param ldc is row stride of `C`- * @param ith is thread id (must be less than `nth`)- * @param nth is number of threads (must be greater than zero)- * @param Atype is GGML data type of `A`- * @param Btype is GGML data type of `B`- * @param Ctype is GGML data type of `C`- * @return true if this function was able to service the matmul request- */-bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,- int Ctype) {-- assert(m >= 0);- assert(n >= 0);- assert(k >= 0);- assert(lda >= k);- assert(ldb >= k);- assert(ldc >= m);- assert(nth > 0);- assert(ith < nth);-- if (Ctype != GGML_TYPE_F32)- return false;+template <typename TC>+bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const void *B, long ldb,+ TC *C, long ldc, int ith, int nth, int task, int Atype, int Btype,+ int precision) {switch (Atype) {@@ -981,22 +949,22 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void#if defined(__AVX512F__)if (k % 16)return false;- tinyBLAS<0, 16, __m512, __m512, float, float, float> tb{- k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 16, __m512, __m512, float, float, TC> tb{+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX__) || defined(__AVX2__)if (k % 8)return false;- tinyBLAS<0, 8, __m256, __m256, float, float, float> tb{- k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 8, __m256, __m256, float, float, TC> tb{+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__ARM_NEON)if (k % 4)return false;- tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, float> tb{- k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, TC> tb{+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else@@ -1009,22 +977,22 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidif (k % 32)return false;if (Btype == GGML_TYPE_F32 && n < 2) {- tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, float> tb{- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, TC> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;}if (Btype != GGML_TYPE_BF16)return false;- tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{- k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC> tb{+ k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX512F__)if (k % 16)return false;- tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, float> tb{- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX2__)@@ -1032,8 +1000,8 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;if (Btype != GGML_TYPE_F32)return false;- tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, float> tb{- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, TC> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__ARM_NEON) && !defined(_MSC_VER)@@ -1041,8 +1009,8 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;if (Btype != GGML_TYPE_F32)return false;- tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, float> tb{- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, TC> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else@@ -1055,15 +1023,15 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidif (k % 16)return false;if (Btype == GGML_TYPE_F32 && n < 2) {- tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, TC> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;}if (Btype != GGML_TYPE_F16)return false;- tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float> tb{- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, TC> tb{+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)@@ -1071,17 +1039,15 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidif (k % 8)return false;if (Btype == GGML_TYPE_F32 && n < 2) {- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith,- nth};+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, TC> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;}if (Btype != GGML_TYPE_F16)return false;- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith,- nth};+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, TC> tb{+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;} else {@@ -1091,14 +1057,25 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidif (n < 2)// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?return false;- if (k % 8)- return false;- if (Btype != GGML_TYPE_F16)- return false;- tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);- return true;+ if (precision == GGML_PREC_F32) {+ if (k % 4)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+ } else {+ if (k % 8)+ return false;+ if (Btype != GGML_TYPE_F16)+ return false;+ tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, TC> tb{+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+ }#elif defined(__ARM_NEON) && !defined(_MSC_VER)if (n < 2)// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?@@ -1107,8 +1084,8 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;if (Btype != GGML_TYPE_F32)return false;- tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else@@ -1120,13 +1097,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidif (Btype != GGML_TYPE_Q8_0)return false;#if defined(__AVX2__) || defined(__AVX512F__)- tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, float> tb{- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, TC> tb{+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__ARM_FEATURE_DOTPROD)- tinyBLAS_Q0_ARM<0, block_q8_0, block_q8_0, float> tb{- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS_Q0_ARM<0, block_q8_0, block_q8_0, TC> tb{+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else@@ -1138,13 +1115,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidif (Btype != GGML_TYPE_Q8_0)return false;#if defined(__AVX2__) || defined(__AVX512F__)- tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, float> tb{- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, TC> tb{+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__ARM_FEATURE_DOTPROD)- tinyBLAS_Q0_ARM<0, block_q4_0, block_q8_0, float> tb{- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tinyBLAS_Q0_ARM<0, block_q4_0, block_q8_0, TC> tb{+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else@@ -1169,7 +1146,65 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void(void)nth;(void)Atype;(void)Btype;- (void)Ctype;+ (void)precision;+}++} // namespace++/**+ * Performs optimized matrix multiplication on CPU.+ *+ * This subroutine may compute C = Aᵀ * B with column major ordering.+ * Despite its name, this isn't a generalized implementation. Work is+ * only performed when a handwritten kernel is written and available.+ * Otherwise the caller should fall back to a general matmul routine.+ *+ * For example, for single-threaded single-precision GEMM you can say+ *+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1,+ * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32,+ * GGML_PREC_DEFAULT);+ *+ * @param m is rows in `A` and `C`+ * @param n is cols in `B` and `C`+ * @param k is cols in `A` and rows in `B`+ * @param A is first input matrix (always transposed)+ * @param lda is row stride of `A`+ * @param B is second input matrix (never transposed)+ * @param ldb is row stride of `B`+ * @param C is input/output array of output matrices+ * @param ldc is row stride of `C`+ * @param ith is thread id (must be less than `nth`)+ * @param nth is number of threads (must be greater than zero)+ * @param Atype is GGML data type of `A`+ * @param Btype is GGML data type of `B`+ * @param Ctype is GGML data type of `C`+ * @param precision may be used to control the internal compute type+ * @return true if this function was able to service the matmul request+ */+bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,+ void *C, long ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype,+ int precision) {++ assert(m >= 0);+ assert(n >= 0);+ assert(k >= 0);+ assert(lda >= k);+ assert(ldb >= k);+ assert(ldc >= m);+ assert(nth > 0);+ assert(ith < nth);++ switch (Ctype) {+ case GGML_TYPE_F32:+ return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (float *)C, ldc, ith, nth, task, Atype,+ Btype, precision);+ case GGML_TYPE_F16:+ return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_fp16_t *)C, ldc, ith, nth, task,+ Atype, Btype, precision);+ default:+ return false;+ }}//@@ -1285,15 +1320,12 @@ class MixMul {assert(params->ith < params->nth);assert(plan->type == GGML_TYPE_I32);- // supported types- if (result->type != GGML_TYPE_F32)- return false;-// check nb01 is convertible to ldaif (weights->nb[1] % ggml_type_size(weights->type))return false;// no support for column strides+if (result->nb[0] != ggml_type_size(result->type))return false;if (thought->nb[0] != ggml_type_size(thought->type))@@ -1301,24 +1333,37 @@ class MixMul {if (weights->nb[0] != ggml_type_size(weights->type))return false;+ // supported output types+ switch (result->type) {+ case GGML_TYPE_F32:+ return mixmuler<float>();+ case GGML_TYPE_F16:+ return mixmuler<ggml_fp16_t>();+ default:+ return false;+ }+ }++ private:+ template <typename TC>+ bool mixmuler() {switch (weights->type) {case GGML_TYPE_F32:if (thought->type != GGML_TYPE_F32)return false;#if defined(__AVX512F__)- return mixmat<16, 1, tinyBLAS<NCB | NCC, 16, __m512, __m512, float, float, float>,- float, float, float>();+ return mixmat<16, 1, tinyBLAS<NCB | NCC, 16, __m512, __m512, float, float, TC>, float,+ float, TC>();#elif defined(__AVX__) || defined(__AVX2__)- return mixmat<8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, float, float, float>, float,- float, float>();+ return mixmat<8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, float, float, TC>, float,+ float, TC>();#elif defined(__SSE__)- return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, __m128, __m128, float, float, float>, float,- float, float>();+ return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, __m128, __m128, float, float, TC>, float,+ float, TC>();#elif defined(__ARM_NEON)- return mixmat<4, 1,- tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, float, float, float>,- float, float, float>();+ return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, float, float, TC>,+ float, float, TC>();#elsereturn false;#endif@@ -1327,22 +1372,22 @@ class MixMul {if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_BF16)return false;#if defined(__AVX512BF16__)- return mixmat<- 32, 1, tinyBLAS<NCB | NCC, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float>,- ggml_bf16_t, ggml_bf16_t, float>();+ return mixmat<32, 1,+ tinyBLAS<NCB | NCC, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC>,+ ggml_bf16_t, ggml_bf16_t, TC>();#elif defined(__AVX512F__)return mixmat<16, 1,- tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, float>,- ggml_bf16_t, ggml_bf16_t, float>();+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, TC>,+ ggml_bf16_t, ggml_bf16_t, TC>();#elif defined(__AVX2__)return mixmat<8, 1,- tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, float>,- ggml_bf16_t, ggml_bf16_t, float>();+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, TC>,+ ggml_bf16_t, ggml_bf16_t, TC>();#elif defined(__ARM_NEON) && !defined(_MSC_VER)return mixmat<4, 1,- tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float>,- ggml_bf16_t, ggml_bf16_t, float>();+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, TC>,+ ggml_bf16_t, ggml_bf16_t, TC>();#elsereturn false;#endif@@ -1352,26 +1397,33 @@ class MixMul {return false;#if defined(__AVX512F__)return mixmat<16, 1,- tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float>,- ggml_fp16_t, ggml_fp16_t, float>();+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, TC>,+ ggml_fp16_t, ggml_fp16_t, TC>();#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)if (X86_HAVE(F16C)) {- return mixmat<- 8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,- ggml_fp16_t, ggml_fp16_t, float>();+ return mixmat<8, 1,+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, TC>,+ ggml_fp16_t, ggml_fp16_t, TC>();} else {return false;}#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)- return mixmat<- 8, 1,- tinyBLAS<NCB | NCC, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float>,- ggml_fp16_t, ggml_fp16_t, float>();+ if (result->op_params[0] == GGML_PREC_F32) {+ return mixmat<+ 4, 1,+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, TC>,+ ggml_fp16_t, ggml_fp16_t, TC>();+ } else {+ return mixmat<+ 8, 1,+ tinyBLAS<NCB | NCC, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, TC>,+ ggml_fp16_t, ggml_fp16_t, TC>();+ }#elif defined(__ARM_NEON) && !defined(_MSC_VER)return mixmat<4, 1,- tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, float>,- ggml_fp16_t, ggml_fp16_t, float>();+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, TC>,+ ggml_fp16_t, ggml_fp16_t, TC>();#elsereturn false;#endif@@ -1380,11 +1432,11 @@ class MixMul {if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)return false;#if defined(__AVX2__) || defined(__AVX512F__)- return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q4_0, block_q8_0, float>,- block_q4_0, block_q8_0, float>();+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q4_0, block_q8_0, TC>,+ block_q4_0, block_q8_0, TC>();#elif defined(__ARM_FEATURE_DOTPROD)- return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0, block_q8_0, float>,- block_q4_0, block_q8_0, float>();+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0, block_q8_0, TC>,+ block_q4_0, block_q8_0, TC>();#elsereturn false;#endif@@ -1393,11 +1445,11 @@ class MixMul {if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)return false;#if defined(__AVX2__) || defined(__AVX512F__)- return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q8_0, block_q8_0, float>,- block_q8_0, block_q8_0, float>();+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q8_0, block_q8_0, TC>,+ block_q8_0, block_q8_0, TC>();#elif defined(__ARM_FEATURE_DOTPROD)- return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0, block_q8_0, float>,- block_q8_0, block_q8_0, float>();+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0, block_q8_0, TC>,+ block_q8_0, block_q8_0, TC>();#elsereturn false;#endif@@ -1407,7 +1459,6 @@ class MixMul {}}- private:template <int KN, int BS, typename BLAS, typename TA, typename TB, typename TC>bool mixmat() {if (cols % KN)