Skip to content

Distribute and run LLMs with a single file.

Upgrade to latest llama.cpp code

Justine Tunney0bdea60a92b1
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
index bd1dde8d..f09c18f5 100644
--- a/llama.cpp/BUILD.mk
+++ b/llama.cpp/BUILD.mk
@@ -43,6 +43,11 @@ o/$(MODE)/llama.cpp/ggml-alloc.o \
o/$(MODE)/llama.cpp/common.o: private \
CCFLAGS += -Os
+o/$(MODE)/llama.cpp/ggml-quants.o: private CXXFLAGS += -Os
+o/$(MODE)/llama.cpp/ggml-quants-amd-avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge
+o/$(MODE)/llama.cpp/ggml-quants-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2
+o/$(MODE)/llama.cpp/ggml-quants-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
+
$(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk
.PHONY: o/$(MODE)/llama.cpp
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafile
index 0cca3351..d6eac66a 100644
--- a/llama.cpp/README.llamafile
+++ b/llama.cpp/README.llamafile
@@ -9,8 +9,8 @@ LICENSE
ORIGIN
https://github.com/ggerganov/llama.cpp/pull/4406/
- 46e12c4692a37bdd31a0432fc5153d7d22bc7f72
- 2024-04-25
+ a8f9b076316e16aadd0791015b3bfd446fe1e904
+ 2024-04-30
LOCAL MODIFICATIONS
@@ -21,9 +21,9 @@ LOCAL MODIFICATIONS
- Add support to main() programs for Cosmo /zip/.args files
- Introduce pledge() SECCOMP sandboxing to improve security
- Call exit() rather than abort() when GGML_ASSERT() fails
+ - Clamp bf16/f32 values before passing to K quantizers
- Make GPU logger callback API safer and less generic
- Write log to /dev/null when main.log fails to open
- - Use _rand64() rather than time() as default seed
- Make main and llava-cli print timings on ctrl-c
- Avoid bind() conflicts on port 8080 w/ server
- Use runtime dispatching for matmul quants
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index 0a777802..e57f083f 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -227,8 +227,54 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
return result;
}
+bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {
+ const char * sep = strchr(data, '=');
+ if (sep == nullptr || sep - data >= 128) {
+ fprintf(stderr, "%s: malformed KV override '%s'\n", __func__, data);
+ return false;
+ }
+ llama_model_kv_override kvo;
+ std::strncpy(kvo.key, data, sep - data);
+ kvo.key[sep - data] = 0;
+ sep++;
+ if (strncmp(sep, "int:", 4) == 0) {
+ sep += 4;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
+ kvo.val_i64 = std::atol(sep);
+ } else if (strncmp(sep, "float:", 6) == 0) {
+ sep += 6;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;
+ kvo.val_f64 = std::atof(sep);
+ } else if (strncmp(sep, "bool:", 5) == 0) {
+ sep += 5;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;
+ if (std::strcmp(sep, "true") == 0) {
+ kvo.val_bool = true;
+ } else if (std::strcmp(sep, "false") == 0) {
+ kvo.val_bool = false;
+ } else {
+ fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data);
+ return false;
+ }
+ } else if (strncmp(sep, "str:", 4) == 0) {
+ sep += 4;
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;
+ if (strlen(sep) > 127) {
+ fprintf(stderr, "%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data);
+ return false;
+ }
+ strncpy(kvo.val_str, sep, 127);
+ kvo.val_str[127] = '\0';
+ } else {
+ fprintf(stderr, "%s: invalid type for KV override '%s'\n", __func__, data);
+ return false;
+ }
+ overrides.emplace_back(std::move(kvo));
+ return true;
+}
+
bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param) {
- llama_sampling_params& sparams = params.sparams;
+ llama_sampling_params & sparams = params.sparams;
if (arg == "--cli") {
return true;
@@ -878,7 +924,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
invalid_param = true;
return true;
}
- params.image = argv[i];
+ params.image.emplace_back(argv[i]);
return true;
}
if (arg == "-i" || arg == "--interactive") {
@@ -933,6 +979,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
params.cont_batching = true;
return true;
}
+ if (arg == "-fa" || arg == "--flash-attn") {
+ params.flash_attn = true;
+ return true;
+ }
if (arg == "--color") {
params.use_color = true;
return true;
@@ -1101,6 +1151,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
params.n_print = std::stoi(argv[i]);
return true;
}
+ if (arg == "--check-tensors") {
+ params.check_tensors = true;
+ return true;
+ }
if (arg == "--ppl-output-type") {
if (++i >= argc) {
invalid_param = true;
@@ -1252,47 +1306,11 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
invalid_param = true;
return true;
}
- char* sep = strchr(argv[i], '=');
- if (sep == nullptr || sep - argv[i] >= 128) {
- fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);
- invalid_param = true;
- return true;
- }
- struct llama_model_kv_override kvo;
- std::strncpy(kvo.key, argv[i], sep - argv[i]);
- kvo.key[sep - argv[i]] = 0;
- sep++;
- if (strncmp(sep, "int:", 4) == 0) {
- sep += 4;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
- kvo.int_value = std::atol(sep);
- }
- else if (strncmp(sep, "float:", 6) == 0) {
- sep += 6;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;
- kvo.float_value = std::atof(sep);
- }
- else if (strncmp(sep, "bool:", 5) == 0) {
- sep += 5;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;
- if (std::strcmp(sep, "true") == 0) {
- kvo.bool_value = true;
- }
- else if (std::strcmp(sep, "false") == 0) {
- kvo.bool_value = false;
- }
- else {
- fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);
- invalid_param = true;
- return true;
- }
- }
- else {
+ if (!parse_kv_override(argv[i], params.kv_overrides)) {
fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);
invalid_param = true;
return true;
}
- params.kv_overrides.push_back(kvo);
return true;
}
#ifndef LOG_DISABLE_LOGS
@@ -1322,6 +1340,29 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
return false;
}
+void gpt_params_handle_model_default(gpt_params & params) {
+ if (!params.hf_repo.empty()) {
+ // short-hand to avoid specifying --hf-file -> default it to --model
+ if (params.hf_file.empty()) {
+ if (params.model.empty()) {
+ throw std::invalid_argument("error: --hf-repo requires either --hf-file or --model\n");
+ }
+ params.hf_file = params.model;
+ } else if (params.model.empty()) {
+ params.model = "models/" + string_split(params.hf_file, '/').back();
+ }
+ } else if (!params.model_url.empty()) {
+ if (params.model.empty()) {
+ auto f = string_split(params.model_url, '#').front();
+ f = string_split(f, '?').front();
+ f = string_split(f, '/').back();
+ params.model = "models/" + f;
+ }
+ } else if (params.model.empty()) {
+ params.model = DEFAULT_MODEL_PATH;
+ }
+}
+
bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
bool invalid_param = false;
std::string arg;
@@ -1350,10 +1391,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
}
- // short-hand to avoid specifying --hf-file -> default it to --model
- if (!params.hf_repo.empty() && params.hf_file.empty()) {
- params.hf_file = params.model;
- }
+ gpt_params_handle_model_default(params);
if (params.escape) {
process_escapes(params.prompt);
@@ -1494,8 +1532,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
printf(" -ns N, --sequences N number of sequences to decode (default: %d)\n", params.n_sequences);
printf(" -ps N, --p-split N speculative decoding split probability (default: %.1f)\n", (double)params.p_split);
printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");
+ printf(" -fa, --flash-attn enable Flash Attention (default: %s)\n", params.flash_attn ? "enabled" : "disabled");
printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA. see examples/llava/README.md\n");
- printf(" --image IMAGE_FILE path to an image file. use with multimodal models\n");
+ printf(" --image IMAGE_FILE path to an image file. use with multimodal models. Specify multiple times for batching\n");
if (llama_supports_mlock()) {
printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");
}
@@ -1548,7 +1587,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
printf(" --control-vector-layer-range START END\n");
printf(" layer range to apply the control vector(s) to, start and end inclusive\n");
printf(" -m FNAME, --model FNAME\n");
- printf(" model path (default: %s)\n", params.model.c_str());
+ printf(" model path (default: models/$filename with filename from --hf-file or --model-url if set, otherwise %s)\n", DEFAULT_MODEL_PATH);
printf(" -md FNAME, --model-draft FNAME\n");
printf(" draft model for speculative decoding (default: unused)\n");
printf(" -mu MODEL_URL, --model-url MODEL_URL\n");
@@ -1565,9 +1604,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
printf(" path to dynamic lookup cache to use for lookup decoding (updated by generation)\n");
printf(" --override-kv KEY=TYPE:VALUE\n");
printf(" advanced option to override model metadata by key. may be specified multiple times.\n");
- printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");
+ printf(" types: int, float, bool, str. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");
printf(" -ptc N, --print-token-count N\n");
printf(" print token count every N tokens (default: %d)\n", params.n_print);
+ printf(" --check-tensors check model tensor data for invalid values\n");
printf("\n");
#ifndef LOG_DISABLE_LOGS
log_print_usage();
@@ -1692,6 +1732,18 @@ std::vector<std::string> string_split(std::string input, char separator) {
return parts;
}
+std::string string_strip(const std::string & str) {
+ size_t start = 0;
+ size_t end = str.size();
+ while (start < end && std::isspace(str[start])) {
+ start++;
+ }
+ while (end > start && std::isspace(str[end - 1])) {
+ end--;
+ }
+ return str.substr(start, end - start);
+}
+
std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names) {
std::unordered_map<std::string, llama_sampler_type> sampler_canonical_name_map {
{"top_k", llama_sampler_type::TOP_K},
@@ -1788,6 +1840,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
mparams.tensor_split = params.tensor_split;
mparams.use_mmap = params.use_mmap;
mparams.use_mlock = params.use_mlock;
+ mparams.check_tensors = params.check_tensors;
if (params.kv_overrides.empty()) {
mparams.kv_overrides = NULL;
} else {
@@ -1852,6 +1905,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param
cparams.cb_eval = params.cb_eval;
cparams.cb_eval_user_data = params.cb_eval_user_data;
cparams.offload_kqv = !params.no_kv_offload;
+ cparams.flash_attn = params.flash_attn;
cparams.type_k = kv_cache_type_from_str(params.cache_type_k);
cparams.type_v = kv_cache_type_from_str(params.cache_type_v);
@@ -1882,59 +1936,75 @@ void llama_batch_add(
#ifdef LLAMA_USE_CURL
-static bool llama_download_file(CURL * curl, const char * url, const char * path) {
+static bool starts_with(const std::string & str, const std::string & prefix) {
+ // While we wait for C++20's std::string::starts_with...
+ return str.rfind(prefix, 0) == 0;
+}
+
+static bool llama_download_file(const std::string & url, const std::string & path) {
+
+ // Initialize libcurl
+ std::unique_ptr<CURL, decltype(&curl_easy_cleanup)> curl(curl_easy_init(), &curl_easy_cleanup);
+ if (!curl) {
+ fprintf(stderr, "%s: error initializing libcurl\n", __func__);
+ return false;
+ }
+
bool force_download = false;
// Set the URL, allow to follow http redirection
- curl_easy_setopt(curl, CURLOPT_URL, url);
- curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
+ curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
#if defined(_WIN32)
// CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of
// operating system. Currently implemented under MS-Windows.
- curl_easy_setopt(curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);
+ curl_easy_setopt(curl.get(), CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);
#endif
// Check if the file already exists locally
struct stat model_file_info;
- auto file_exists = (stat(path, &model_file_info) == 0);
-
- // If the file exists, check for ${path_model}.etag or ${path_model}.lastModified files
- char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};
- char etag_path[PATH_MAX] = {0};
- snprintf(etag_path, sizeof(etag_path), "%s.etag", path);
+ auto file_exists = (stat(path.c_str(), &model_file_info) == 0);
- char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};
- char last_modified_path[PATH_MAX] = {0};
- snprintf(last_modified_path, sizeof(last_modified_path), "%s.lastModified", path);
+ // If the file exists, check its JSON metadata companion file.
+ std::string metadata_path = path + ".json";
+ nlohmann::json metadata;
+ std::string etag;
+ std::string last_modified;
if (file_exists) {
- auto * f_etag = fopen(etag_path, "r");
- if (f_etag) {
- if (!fgets(etag, sizeof(etag), f_etag)) {
- fprintf(stderr, "%s: unable to read file %s\n", __func__, etag_path);
- } else {
- fprintf(stderr, "%s: previous file found %s: %s\n", __func__, etag_path, etag);
- }
- fclose(f_etag);
- }
-
- auto * f_last_modified = fopen(last_modified_path, "r");
- if (f_last_modified) {
- if (!fgets(last_modified, sizeof(last_modified), f_last_modified)) {
- fprintf(stderr, "%s: unable to read file %s\n", __func__, last_modified_path);
- } else {
- fprintf(stderr, "%s: previous file found %s: %s\n", __func__, last_modified_path,
- last_modified);
+ // Try and read the JSON metadata file (note: stream autoclosed upon exiting this block).
+ std::ifstream metadata_in(metadata_path);
+ if (metadata_in.good()) {
+ try {
+ metadata_in >> metadata;
+ fprintf(stderr, "%s: previous metadata file found %s: %s\n", __func__, metadata_path.c_str(), metadata.dump().c_str());
+ if (metadata.contains("url") && metadata["url"].is_string()) {
+ auto previous_url = metadata["url"].get<std::string>();
+ if (previous_url != url) {
+ fprintf(stderr, "%s: Model URL mismatch: %s != %s\n", __func__, url.c_str(), previous_url.c_str());
+ return false;
+ }
+ }
+ if (metadata.contains("etag") && metadata["etag"].is_string()) {
+ etag = metadata["etag"];
+ }
+ if (metadata.contains("lastModified") && metadata["lastModified"].is_string()) {
+ last_modified = metadata["lastModified"];
+ }
+ } catch (const nlohmann::json::exception & e) {
+ fprintf(stderr, "%s: error reading metadata file %s: %s\n", __func__, metadata_path.c_str(), e.what());
+ return false;
}
- fclose(f_last_modified);
}
+ } else {
+ fprintf(stderr, "%s: no previous model file found %s\n", __func__, path.c_str());
}
// Send a HEAD request to retrieve the etag and last-modified headers
struct llama_load_model_from_url_headers {
- char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};
- char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};
+ std::string etag;
+ std::string last_modified;
};
llama_load_model_from_url_headers headers;
{
@@ -1942,38 +2012,37 @@ static bool llama_download_file(CURL * curl, const char * url, const char * path
auto header_callback = [](char * buffer, size_t /*size*/, size_t n_items, void * userdata) -> size_t {
llama_load_model_from_url_headers *headers = (llama_load_model_from_url_headers *) userdata;
- // Convert header field name to lowercase
- for (size_t i = 0; i < n_items && buffer[i] != ':'; ++i) {
- buffer[i] = tolower(buffer[i]);
- }
-
- const char * etag_prefix = "etag: ";
- if (strncmp(buffer, etag_prefix, strlen(etag_prefix)) == 0) {
- strncpy(headers->etag, buffer + strlen(etag_prefix), n_items - strlen(etag_prefix) - 2); // Remove CRLF
- }
-
- const char * last_modified_prefix = "last-modified: ";
- if (strncmp(buffer, last_modified_prefix, strlen(last_modified_prefix)) == 0) {
- strncpy(headers->last_modified, buffer + strlen(last_modified_prefix),
- n_items - strlen(last_modified_prefix) - 2); // Remove CRLF
+ static std::regex header_regex("([^:]+): (.*)\r\n");
+ static std::regex etag_regex("ETag", std::regex_constants::icase);
+ static std::regex last_modified_regex("Last-Modified", std::regex_constants::icase);
+
+ std::string header(buffer, n_items);
+ std::smatch match;
+ if (std::regex_match(header, match, header_regex)) {
+ const std::string & key = match[1];
+ const std::string & value = match[2];
+ if (std::regex_match(key, match, etag_regex)) {
+ headers->etag = value;
+ } else if (std::regex_match(key, match, last_modified_regex)) {
+ headers->last_modified = value;
+ }
}
return n_items;
};
- curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); // will trigger the HEAD verb
- curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); // hide head request progress
- curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, static_cast<CURLOPT_HEADERFUNCTION_PTR>(header_callback));
- curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);
+ curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 1L); // will trigger the HEAD verb
+ curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1L); // hide head request progress
+ curl_easy_setopt(curl.get(), CURLOPT_HEADERFUNCTION, static_cast<CURLOPT_HEADERFUNCTION_PTR>(header_callback));
+ curl_easy_setopt(curl.get(), CURLOPT_HEADERDATA, &headers);
- CURLcode res = curl_easy_perform(curl);
+ CURLcode res = curl_easy_perform(curl.get());
if (res != CURLE_OK) {
- curl_easy_cleanup(curl);
fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));
return false;
}
long http_code = 0;
- curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+ curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &http_code);
if (http_code != 200) {
// HEAD not supported, we don't know if the file has changed
// force trigger downloading
@@ -1982,28 +2051,30 @@ static bool llama_download_file(CURL * curl, const char * url, const char * path
}
}
- // If the ETag or the Last-Modified headers are different: trigger a new download
- bool should_download = !file_exists
- || force_download
- || (strlen(headers.etag) > 0 && strcmp(etag, headers.etag) != 0)
- || (strlen(headers.last_modified) > 0 && strcmp(last_modified, headers.last_modified) != 0);
+ bool should_download = !file_exists || force_download;
+ if (!should_download) {
+ if (!etag.empty() && etag != headers.etag) {
+ fprintf(stderr, "%s: ETag header is different (%s != %s): triggering a new download\n", __func__, etag.c_str(), headers.etag.c_str());
+ should_download = true;
+ } else if (!last_modified.empty() && last_modified != headers.last_modified) {
+ fprintf(stderr, "%s: Last-Modified header is different (%s != %s): triggering a new download\n", __func__, last_modified.c_str(), headers.last_modified.c_str());
+ should_download = true;
+ }
+ }
if (should_download) {
- char path_temporary[PATH_MAX] = {0};
- snprintf(path_temporary, sizeof(path_temporary), "%s.downloadInProgress", path);
+ std::string path_temporary = path + ".downloadInProgress";
if (file_exists) {
- fprintf(stderr, "%s: deleting previous downloaded file: %s\n", __func__, path);
- if (remove(path) != 0) {
- curl_easy_cleanup(curl);
- fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path);
+ fprintf(stderr, "%s: deleting previous downloaded file: %s\n", __func__, path.c_str());
+ if (remove(path.c_str()) != 0) {
+ fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path.c_str());
return false;
}
}
// Set the output file
- auto * outfile = fopen(path_temporary, "wb");
+ std::unique_ptr<FILE, decltype(&fclose)> outfile(fopen(path_temporary.c_str(), "wb"), fclose);
if (!outfile) {
- curl_easy_cleanup(curl);
- fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path);
+ fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path.c_str());
return false;
}
@@ -2011,12 +2082,12 @@ static bool llama_download_file(CURL * curl, const char * url, const char * path
auto write_callback = [](void * data, size_t size, size_t nmemb, void * fd) -> size_t {
return fwrite(data, size, nmemb, (FILE *)fd);
};
- curl_easy_setopt(curl, CURLOPT_NOBODY, 0L);
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile);
+ curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 0L);
+ curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));
+ curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, outfile.get());
// display download progress
- curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
+ curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 0L);
// helper function to hide password in URL
auto llama_download_hide_password_in_url = [](const std::string & url) -> std::string {
@@ -2035,51 +2106,34 @@ static bool llama_download_file(CURL * curl, const char * url, const char * path
// start the download
fprintf(stderr, "%s: downloading from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__,
- llama_download_hide_password_in_url(url).c_str(), path, headers.etag, headers.last_modified);
- auto res = curl_easy_perform(curl);
+ llama_download_hide_password_in_url(url).c_str(), path.c_str(), headers.etag.c_str(), headers.last_modified.c_str());
+ auto res = curl_easy_perform(curl.get());
if (res != CURLE_OK) {
- fclose(outfile);
- curl_easy_cleanup(curl);
fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));
return false;
}
long http_code = 0;
- curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);
+ curl_easy_getinfo (curl.get(), CURLINFO_RESPONSE_CODE, &http_code);
if (http_code < 200 || http_code >= 400) {
- fclose(outfile);
- curl_easy_cleanup(curl);
fprintf(stderr, "%s: invalid http status code received: %ld\n", __func__, http_code);
return false;
}
- // Clean up
- fclose(outfile);
+ // Causes file to be closed explicitly here before we rename it.
+ outfile.reset();
- // Write the new ETag to the .etag file
- if (strlen(headers.etag) > 0) {
- auto * etag_file = fopen(etag_path, "w");
- if (etag_file) {
- fputs(headers.etag, etag_file);
- fclose(etag_file);
- fprintf(stderr, "%s: file etag saved %s: %s\n", __func__, etag_path, headers.etag);
- }
- }
-
- // Write the new lastModified to the .etag file
- if (strlen(headers.last_modified) > 0) {
- auto * last_modified_file = fopen(last_modified_path, "w");
- if (last_modified_file) {
- fputs(headers.last_modified, last_modified_file);
- fclose(last_modified_file);
- fprintf(stderr, "%s: file last modified saved %s: %s\n", __func__, last_modified_path,
- headers.last_modified);
- }
- }
+ // Write the updated JSON metadata file.
+ metadata.update({
+ {"url", url},
+ {"etag", headers.etag},
+ {"lastModified", headers.last_modified}
+ });
+ std::ofstream(metadata_path) << metadata.dump(4);
+ fprintf(stderr, "%s: file metadata saved: %s\n", __func__, metadata_path.c_str());
- if (rename(path_temporary, path) != 0) {
- curl_easy_cleanup(curl);
- fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_temporary, path);
+ if (rename(path_temporary.c_str(), path.c_str()) != 0) {
+ fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str());
return false;
}
}
@@ -2097,15 +2151,7 @@ struct llama_model * llama_load_model_from_url(
return NULL;
}
- // Initialize libcurl
- auto * curl = curl_easy_init();
-
- if (!curl) {
- fprintf(stderr, "%s: error initializing libcurl\n", __func__);
- return NULL;
- }
-
- if (!llama_download_file(curl, model_url, path_model)) {
+ if (!llama_download_file(model_url, path_model)) {
return NULL;
}
@@ -2119,7 +2165,6 @@ struct llama_model * llama_load_model_from_url(
auto * ctx_gguf = gguf_init_from_file(path_model, gguf_params);
if (!ctx_gguf) {
fprintf(stderr, "\n%s: failed to load input GGUF from %s\n", __func__, path_model);
- curl_easy_cleanup(curl);
return NULL;
}
@@ -2131,8 +2176,6 @@ struct llama_model * llama_load_model_from_url(
gguf_free(ctx_gguf);
}
- curl_easy_cleanup(curl);
-
if (n_split > 1) {
char split_prefix[PATH_MAX] = {0};
char split_url_prefix[LLAMA_CURL_MAX_URL_LENGTH] = {0};
@@ -2163,11 +2206,7 @@ struct llama_model * llama_load_model_from_url(
char split_url[LLAMA_CURL_MAX_URL_LENGTH] = {0};
llama_split_path(split_url, sizeof(split_url), split_url_prefix, download_idx, n_split);
- auto * curl = curl_easy_init();
- bool res = llama_download_file(curl, split_url, split_path);
- curl_easy_cleanup(curl);
-
- return res;
+ return llama_download_file(split_url, split_path);
}, idx));
}
@@ -2654,7 +2693,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l
fprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau);
fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta);
fprintf(stream, "mlock: %s # default: false\n", params.use_mlock ? "true" : "false");
- fprintf(stream, "model: %s # default: models/7B/ggml-model.bin\n", params.model.c_str());
+ fprintf(stream, "model: %s # default: %s\n", params.model.c_str(), DEFAULT_MODEL_PATH);
fprintf(stream, "model_draft: %s # default:\n", params.model_draft.c_str());
fprintf(stream, "multiline_input: %s # default: false\n", params.multiline_input ? "true" : "false");
fprintf(stream, "n_gpu_layers: %d # default: -1\n", params.n_gpu_layers);
@@ -2689,6 +2728,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l
fprintf(stream, "seed: %u # default: -1 (random seed)\n", params.seed);
fprintf(stream, "simple_io: %s # default: false\n", params.simple_io ? "true" : "false");
fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false");
+ fprintf(stream, "flash_attn: %s # default: false\n", params.flash_attn ? "true" : "false");
fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp);
const std::vector<float> tensor_split_vector(params.tensor_split, params.tensor_split + llama_max_devices());
diff --git a/llama.cpp/common.h b/llama.cpp/common.h
index 75fcc901..2cbd376c 100644
--- a/llama.cpp/common.h
+++ b/llama.cpp/common.h
@@ -36,6 +36,8 @@
tinylog(__func__, ": llamafile version " LLAMAFILE_VERSION_STRING "\n", NULL); \
} while(0)
+#define DEFAULT_MODEL_PATH "models/7B/ggml-model-f16.gguf"
+
// build info
extern int LLAMA_BUILD_NUMBER;
extern char const *LLAMA_COMMIT;
@@ -97,7 +99,7 @@ struct gpt_params {
// // sampling parameters
struct llama_sampling_params sparams;
- std::string model = "models/7B/ggml-model-f16.gguf"; // model path
+ std::string model = ""; // model path
std::string model_draft = ""; // draft model for speculative decoding
std::string model_alias = "unknown"; // model alias
std::string model_url = ""; // model url to download
@@ -138,7 +140,7 @@ struct gpt_params {
bool multiple_choice = false; // compute TruthfulQA score over random tasks from datafile supplied in prompt
size_t multiple_choice_tasks = 0; // number of tasks to use when computing the TruthfulQA score. If 0, all tasks will be computed
- bool kl_divergence = false; // compute KL-divergence
+ bool kl_divergence = false; // compute KL divergence
bool random_prompt = false; // do not randomize prompt if none provided
bool use_color = false; // use color to distinguish generations and inputs
@@ -153,6 +155,7 @@ struct gpt_params {
bool multiline_input = false; // reverse the usage of `\`
bool simple_io = false; // improves compatibility with subprocesses and limited consoles
bool cont_batching = true; // insert new sequences for decoding on-the-fly
+ bool flash_attn = false; // flash attention
bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix
bool ignore_eos = false; // ignore generated EOS tokens
@@ -166,15 +169,20 @@ struct gpt_params {
bool dump_kv_cache = false; // dump the KV cache contents for debugging purposes
bool no_kv_offload = false; // disable KV offloading
bool warmup = true; // warmup run
+ bool check_tensors = false; // validate tensor data
std::string cache_type_k = "f16"; // KV cache data type for the K
std::string cache_type_v = "f16"; // KV cache data type for the V
// multimodal models (see examples/llava)
- std::string mmproj = ""; // path to multimodal projector
- std::string image = ""; // path to an image file
+ std::string mmproj = ""; // path to multimodal projector
+ std::vector<std::string> image; // path to image file(s)
};
+void gpt_params_handle_model_default(gpt_params & params);
+
+bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides);
+
bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params);
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
@@ -198,6 +206,7 @@ bool validate_file_name(const std::string & filename);
std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);
std::vector<llama_sampler_type> sampler_types_from_chars(const std::string & names_string);
std::vector<std::string> string_split(std::string input, char separator);
+std::string string_strip(const std::string & str);
std::string sampler_type_to_name_string(llama_sampler_type sampler_type);
//
diff --git a/llama.cpp/ggml-backend-impl.h b/llama.cpp/ggml-backend-impl.h
index 4871987f..f02379c7 100644
--- a/llama.cpp/ggml-backend-impl.h
+++ b/llama.cpp/ggml-backend-impl.h
@@ -180,6 +180,7 @@ extern "C" {
bool (*GGML_CALL ggml_backend_buffer_is_host)(ggml_backend_buffer_t);
bool (*GGML_CALL ggml_guid_matches)(ggml_guid_t, ggml_guid_t);
bool (*GGML_CALL ggml_is_empty)(const struct ggml_tensor *);
+ bool (*GGML_CALL ggml_are_same_shape)(const struct ggml_tensor *, const struct ggml_tensor *);
};
#ifdef __cplusplus
diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.c
index 749bddc2..c32aa53f 100644
--- a/llama.cpp/ggml-backend.c
+++ b/llama.cpp/ggml-backend.c
@@ -1790,12 +1790,14 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {
void ggml_backend_sched_reset(ggml_backend_sched_t sched) {
// reset state for the next run
- size_t hash_size = sched->hash_set.size;
- memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); // NOLINT
- memset(sched->tensor_backend_id, -1, sizeof(sched->tensor_backend_id[0]) * hash_size);
- memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size);
+ if (!sched->is_reset) {
+ size_t hash_size = sched->hash_set.size;
+ memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); // NOLINT
+ memset(sched->tensor_backend_id, -1, sizeof(sched->tensor_backend_id[0]) * hash_size);
+ memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size);
- sched->is_reset = true;
+ sched->is_reset = true;
+ }
sched->is_alloc = false;
}
@@ -2161,6 +2163,7 @@ static const struct ggml_backend_api kGgmlBackendApi = {
ggml_backend_buffer_is_host,
ggml_guid_matches,
ggml_is_empty,
+ ggml_are_same_shape,
};
const struct ggml_backend_api *ggml_backend_api(void) {
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index 09c73c6a..83b1dc80 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -1,9 +1,8 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-static void ggml_cuda_print(int, const char *, ...);
-
#include <algorithm>
+#include <stdarg.h>
#include <array>
#include <atomic>
#include <cassert>
@@ -17,9 +16,10 @@ static void ggml_cuda_print(int, const char *, ...);
#include <map>
#include <memory>
#include <mutex>
+#include <stdint.h>
+#include <stdio.h>
#include <string>
#include <vector>
-#include <stdarg.h>
#include "ggml.h"
#include "ggml-cuda.h"
@@ -298,6 +298,7 @@ GGML_CALL bool ggml_cuda_link(const struct ggml_backend_api *backend_api) {
#define CC_PASCAL 600
#define MIN_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products
#define CC_VOLTA 700
+#define CC_AMPERE 800
#define CC_OFFSET_AMD 1000000
#define CC_RDNA1 (CC_OFFSET_AMD + 1010)
#define CC_RDNA2 (CC_OFFSET_AMD + 1030)
@@ -427,7 +428,6 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {
return a;
}
-#ifdef GGML_CUDA_F16
static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL
#pragma unroll
@@ -440,7 +440,6 @@ static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
NO_DEVICE_CODE;
#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL
}
-#endif // GGML_CUDA_F16
static __device__ __forceinline__ float warp_reduce_max(float x) {
#pragma unroll
@@ -450,19 +449,26 @@ static __device__ __forceinline__ float warp_reduce_max(float x) {
return x;
}
-//static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {
-//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX
-//#pragma unroll
-// for (int mask = 16; mask > 0; mask >>= 1) {
-// x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));
-// }
-// return x;
-//#else
-// GGML_UNUSED(x);
-// NO_DEVICE_CODE;
-//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX
-//}
+static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));
+ }
+ return x;
+#else
+ GGML_UNUSED(x);
+ NO_DEVICE_CODE;
+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX
+}
+#if CUDART_VERSION < 12000
+static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {
+ const uint32_t mask_low = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b)));
+ const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b)));
+ return mask_low | mask_high;
+}
+#endif // CUDART_VERSION < 12000
#if defined(GGML_USE_HIPBLAS)
#define __CUDA_ARCH__ 1300
@@ -547,6 +553,15 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {
}
#endif // defined(GGML_USE_HIPBLAS)
+#define FP16_AVAILABLE defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) ? \
+ defined(RDNA1) || defined(RDNA2) || defined(RDNA3) : __CUDA_ARCH__ >= CC_PASCAL
+
+#define FP16_MMA_AVAILABLE !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA
+
+#if FP16_MMA_AVAILABLE
+#include <mma.h>
+#endif
+
// TODO: move to ggml-common.h
static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
@@ -560,6 +575,7 @@ struct ggml_cuda_device_info {
struct cuda_device_info {
int cc; // compute capability
+ int nsm; // number of streaming multiprocessors
size_t smpb; // max. shared memory per block
bool vmm; // virtual memory support
size_t vmm_granularity; // granularity of virtual memory
@@ -772,6 +788,8 @@ void ggml_cuda_op_dequantize_mul_mat_vec(
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
const int64_t src1_padded_row_size, cudaStream_t stream);
+void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
#define CUDA_GET_ROWS_BLOCK_SIZE 256
void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
@@ -2849,16 +2867,16 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {
- const int64_t i = 2*(blockDim.x*blockIdx.x + threadIdx.x);
+ const int64_t i = (int64_t)2*(blockDim.x*blockIdx.x + threadIdx.x);
if (i >= k) {
return;
}
const int64_t ib = i/qk; // block index
- const int iqs = (i%qk)/qr; // quant index
- const int iybs = i - i%qk; // y block start index
- const int y_offset = qr == 1 ? 1 : qk/2;
+ const int64_t iqs = (i%qk)/qr; // quant index
+ const int64_t iybs = i - i%qk; // y block start index
+ const int64_t y_offset = qr == 1 ? 1 : qk/2;
// dequantize
dfloat2 v;
@@ -2873,7 +2891,7 @@ static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, h
#if __CUDA_ARCH__ >= CC_PASCAL
constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;
- const int i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;
+ const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;
const int * x0 = ((int *) vx) + blockIdx.x * nint;
half2 * y2 = (half2 *) (y + i0);
@@ -2917,9 +2935,9 @@ static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t
const int64_t i = blockIdx.x;
// assume 32 threads
- const int tid = threadIdx.x;
- const int il = tid/8;
- const int ir = tid%8;
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
const int64_t ib = 8*i + ir;
if (ib >= nb32) {
return;
@@ -2945,9 +2963,9 @@ static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t
const int64_t i = blockIdx.x;
// assume 32 threads
- const int tid = threadIdx.x;
- const int il = tid/8;
- const int ir = tid%8;
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
const int64_t ib = 8*i + ir;
if (ib >= nb32) {
return;
@@ -2971,14 +2989,14 @@ static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t
template<typename dst_t>
static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_q2_K * x = (const block_q2_K *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int n = tid/32;
- const int l = tid - 32*n;
- const int is = 8*n + l/16;
+ const int64_t n = tid/32;
+ const int64_t l = tid - 32*n;
+ const int64_t is = 8*n + l/16;
const uint8_t q = x[i].qs[32*n + l];
dst_t * y = yy + i*QK_K + 128*n;
@@ -2990,8 +3008,8 @@ static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t
y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);
y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);
#else
- const int is = tid/16; // 0 or 1
- const int il = tid%16; // 0...15
+ const int64_t is = tid/16; // 0 or 1
+ const int64_t il = tid%16; // 0...15
const uint8_t q = x[i].qs[il] >> (2*is);
dst_t * y = yy + i*QK_K + 16*is + il;
float dall = __low2half(x[i].dm);
@@ -3005,19 +3023,19 @@ static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t
template<typename dst_t>
static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_q3_K * x = (const block_q3_K *) vx;
#if QK_K == 256
- const int r = threadIdx.x/4;
- const int tid = r/2;
- const int is0 = r%2;
- const int l0 = 16*is0 + 4*(threadIdx.x%4);
- const int n = tid / 4;
- const int j = tid - 4*n;
+ const int64_t r = threadIdx.x/4;
+ const int64_t tid = r/2;
+ const int64_t is0 = r%2;
+ const int64_t l0 = 16*is0 + 4*(threadIdx.x%4);
+ const int64_t n = tid / 4;
+ const int64_t j = tid - 4*n;
uint8_t m = 1 << (4*n + j);
- int is = 8*n + 2*j + is0;
+ int64_t is = 8*n + 2*j + is0;
int shift = 2*j;
int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :
@@ -3033,11 +3051,11 @@ static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t
for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));
#else
- const int tid = threadIdx.x;
- const int is = tid/16; // 0 or 1
- const int il = tid%16; // 0...15
- const int im = il/8; // 0...1
- const int in = il%8; // 0...7
+ const int64_t tid = threadIdx.x;
+ const int64_t is = tid/16; // 0 or 1
+ const int64_t il = tid%16; // 0...15
+ const int64_t im = il/8; // 0...1
+ const int64_t in = il%8; // 0...7
dst_t * y = yy + i*QK_K + 16*is + il;
@@ -3071,15 +3089,15 @@ template<typename dst_t>
static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
const block_q4_K * x = (const block_q4_K *) vx;
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
#if QK_K == 256
// assume 32 threads
- const int tid = threadIdx.x;
- const int il = tid/8;
- const int ir = tid%8;
- const int is = 2*il;
- const int n = 4;
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
+ const int64_t is = 2*il;
+ const int64_t n = 4;
dst_t * y = yy + i*QK_K + 64*il + n*ir;
@@ -3098,7 +3116,7 @@ static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t
y[l +32] = d2 * (q[l] >> 4) - m2;
}
#else
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
const uint8_t * q = x[i].qs;
dst_t * y = yy + i*QK_K;
const float d = (float)x[i].dm[0];
@@ -3112,14 +3130,14 @@ template<typename dst_t>
static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
const block_q5_K * x = (const block_q5_K *) vx;
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
#if QK_K == 256
// assume 64 threads - this is very slightly better than the one below
- const int tid = threadIdx.x;
- const int il = tid/16; // il is in 0...3
- const int ir = tid%16; // ir is in 0...15
- const int is = 2*il; // is is in 0...6
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/16; // il is in 0...3
+ const int64_t ir = tid%16; // ir is in 0...15
+ const int64_t is = 2*il; // is is in 0...6
dst_t * y = yy + i*QK_K + 64*il + 2*ir;
@@ -3142,11 +3160,11 @@ static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t
y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;
y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;
#else
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
const uint8_t q = x[i].qs[tid];
- const int im = tid/8; // 0...3
- const int in = tid%8; // 0...7
- const int is = tid/16; // 0 or 1
+ const int64_t im = tid/8; // 0...3
+ const int64_t in = tid%8; // 0...7
+ const int64_t is = tid/16; // 0 or 1
const uint8_t h = x[i].qh[in] >> im;
const float d = x[i].d;
dst_t * y = yy + i*QK_K + tid;
@@ -3203,13 +3221,13 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t
template<typename dst_t>
static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq2_xxs * x = (const block_iq2_xxs *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
const uint16_t * q2 = x[i].qs + 4*ib;
const uint8_t * aux8 = (const uint8_t *)q2;
@@ -3227,13 +3245,13 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, ds
template<typename dst_t>
static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq2_xs * x = (const block_iq2_xs *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
const uint16_t * q2 = x[i].qs + 4*ib;
const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));
@@ -3249,13 +3267,13 @@ static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst
template<typename dst_t>
static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq2_s * x = (const block_iq2_s *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));
const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
@@ -3270,13 +3288,13 @@ static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_
template<typename dst_t>
static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq3_xxs * x = (const block_iq3_xxs *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
const uint8_t * q3 = x[i].qs + 8*ib;
const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;
@@ -3298,13 +3316,13 @@ static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, ds
template<typename dst_t>
static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq3_s * x = (const block_iq3_s *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
const uint8_t * qs = x[i].qs + 8*ib;
const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));
@@ -3324,13 +3342,13 @@ static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_
template<typename dst_t>
static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq1_s * x = (const block_iq1_s *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;
const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);
@@ -3350,18 +3368,18 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_
template<typename dst_t>
static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq1_m * x = (const block_iq1_m *) vx;
- const int tid = threadIdx.x;
+ const int64_t tid = threadIdx.x;
#if QK_K == 256
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
const uint16_t * sc = (const uint16_t *)x[i].scales;
iq1m_scale_t scale;
scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
- const int ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);
+ const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);
const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);
const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;
uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
@@ -3381,12 +3399,12 @@ static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_
template<typename dst_t>
static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);
- const int tid = threadIdx.x;
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 4*il;
const uint8_t * q4 = x[ib].qs + 4*il;
const float d = (float)x[ib].d;
@@ -3400,12 +3418,12 @@ static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst
#if QK_K != 64
template<typename dst_t>
static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
- const int i = blockIdx.x;
+ const int64_t i = blockIdx.x;
const block_iq4_xs * x = (const block_iq4_xs *)vx;
- const int tid = threadIdx.x;
- const int il = tid/8; // 0...3
- const int ib = tid%8; // 0...7
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
dst_t * y = yy + i*QK_K + 32*ib + 4*il;
const uint8_t * q4 = x[i].qs + 16*ib + 4*il;
const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);
@@ -4978,6 +4996,942 @@ void ggml_cuda_op_dequantize_mul_mat_vec(
GGML_UNUSED(src1_padded_row_size);
}
+#define FATTN_KQ_STRIDE 256
+#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction.
+#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs.
+
+template<int D, int parallel_blocks> // D == head size
+__launch_bounds__(((D + WARP_SIZE - 1) / WARP_SIZE)*WARP_SIZE, 1)
+static __global__ void flash_attn_vec_ext_f16(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3) {
+#if FP16_AVAILABLE
+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.
+
+ const int ic = blockIdx.x / parallel_blocks; // Index of the Q/QKV column to work on.
+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.
+
+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic);
+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));
+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape
+ const half * maskh = (const half *) mask + ne11*ic;
+
+ const int stride_KV = nb11 / sizeof(half);
+ const int stride_KV2 = nb11 / sizeof(half2);
+
+ constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE;
+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;
+ __builtin_assume(tid < nwarps*WARP_SIZE);
+
+ __shared__ half KQ[nwarps*WARP_SIZE];
+ KQ[tid] = -INFINITY;
+ half2 * KQ2 = (half2 *) KQ;
+
+ half kqmax = -HALF_MAX_HALF;
+ half kqsum = 0.0f;
+
+ __shared__ half kqmax_shared[WARP_SIZE];
+ __shared__ half kqsum_shared[WARP_SIZE];
+ if (threadIdx.y == 0) {
+ kqmax_shared[threadIdx.x] = -HALF_MAX_HALF;
+ kqsum_shared[threadIdx.x] = 0.0f;
+ }
+ __syncthreads();
+
+ // Convert Q to half2 and store in registers:
+ half2 Q_h2[(D/2 + WARP_SIZE - 1) / WARP_SIZE];
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {
+ break;
+ }
+
+ Q_h2[i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(Q_f2[i].x, Q_f2[i].y);
+ }
+
+ half2 VKQ = make_half2(0.0f, 0.0f); // Each thread calculates a single VKQ value.
+
+ const int k_start = parallel_blocks == 1 ? 0 : ip*D;
+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) {
+ // Calculate KQ tile and keep track of new maximum KQ values:
+ half kqmax_new = kqmax;
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) {
+ const int i_KQ = i_KQ_0 + threadIdx.y;
+
+ if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) {
+ break;
+ }
+
+ half2 sum2 = make_half2(0.0f, 0.0f);
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {
+ const int k_KQ = k_KQ_0 + threadIdx.x;
+ if (k_KQ_0 + WARP_SIZE > D/2 && k_KQ >= D/2) {
+ break;
+ }
+
+ const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ];
+ sum2 += K_ik * Q_h2[k_KQ_0/WARP_SIZE];
+ }
+
+ sum2 = warp_reduce_sum(sum2);
+ half sum = __low2half(sum2) + __high2half(sum2);
+ sum += mask ? maskh[k_VKQ_0 + i_KQ] : __float2half(0.0f);
+ kqmax_new = __hmax(kqmax_new, sum);
+ if (threadIdx.x == 0) {
+ KQ[i_KQ] = sum;
+ }
+ }
+
+ kqmax_new = warp_reduce_max(kqmax_new);
+ if (threadIdx.x == 0) {
+ kqmax_shared[threadIdx.y] = kqmax_new;
+ }
+ __syncthreads();
+ kqmax_new = kqmax_shared[threadIdx.x];
+ kqmax_new = warp_reduce_max(kqmax_new);
+
+ const half KQ_max_scale = hexp(kqmax - kqmax_new);
+ kqmax = kqmax_new;
+
+ const half val = hexp(KQ[tid] - kqmax);
+ kqsum = kqsum*KQ_max_scale + val;
+ KQ[tid] = val;
+
+ VKQ *= __half2half2(KQ_max_scale);
+
+ __syncthreads();
+
+ if (tid < D) {
+#pragma unroll
+ for (int k0 = 0; k0 < D; k0 += 2) {
+ if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k0 >= ne11) {
+ break;
+ }
+
+ half2 V_k;
+ reinterpret_cast<half&>(V_k.x) = V_h[(k_VKQ_0 + k0 + 0)*stride_KV + tid];
+ reinterpret_cast<half&>(V_k.y) = V_h[(k_VKQ_0 + k0 + 1)*stride_KV + tid];
+ VKQ += V_k*KQ2[k0/2];
+ }
+ }
+
+ __syncthreads();
+ }
+
+ if (tid >= D) {
+ kqsum = 0.0f;
+ }
+
+ kqsum = warp_reduce_sum(kqsum);
+ if (threadIdx.x == 0) {
+ kqsum_shared[threadIdx.y] = kqsum;
+ }
+ __syncthreads();
+ kqsum = kqsum_shared[threadIdx.x];
+ kqsum = warp_reduce_sum(kqsum);
+
+ if (tid >= D) {
+ return;
+ }
+
+ half dst_val = (__low2half(VKQ) + __high2half(VKQ));
+ if (parallel_blocks == 1) {
+ dst_val /= kqsum;
+ }
+ dst[D*gridDim.y*blockIdx.x + D*blockIdx.y + tid] = dst_val;
+
+ if (parallel_blocks == 1 || tid != 0) {
+ return;
+ }
+ dst_meta[ic*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax, kqsum);
+#else
+ NO_DEVICE_CODE;
+#endif // FP16_AVAILABLE
+}
+
+// D == head size, VKQ_stride == num VKQ rows calculated in parallel:
+template<int D, int ncols, int nwarps, int VKQ_stride, int parallel_blocks, typename KQ_acc_t>
+__launch_bounds__(nwarps*WARP_SIZE, 1)
+static __global__ void flash_attn_ext_f16(
+ const char * __restrict__ Q,
+ const char * __restrict__ K,
+ const char * __restrict__ V,
+ const char * __restrict__ mask,
+ float * __restrict__ dst,
+ float2 * __restrict__ dst_meta,
+ const float scale,
+ const int ne00,
+ const int ne01,
+ const int ne02,
+ const int ne03,
+ const int ne10,
+ const int ne11,
+ const int ne12,
+ const int ne13,
+ const int ne31,
+ const int nb31,
+ const int nb01,
+ const int nb02,
+ const int nb03,
+ const int nb11,
+ const int nb12,
+ const int nb13,
+ const int ne0,
+ const int ne1,
+ const int ne2,
+ const int ne3) {
+#if FP16_MMA_AVAILABLE
+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.
+
+ const int ic0 = ncols*(blockIdx.x / parallel_blocks); // Index of the first Q/QKV column to work on.
+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.
+
+ static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE.");
+ static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16.");
+ constexpr int frag_m = ncols == 8 ? 32 : 16;
+ constexpr int frag_n = ncols == 8 ? 8 : 16;
+ static_assert(D % frag_m == 0, "If ncols == 8 then D % frag_m must be 0.");
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::row_major> frag_a_K;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_a_V;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_b;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, KQ_acc_t> frag_c_KQ;
+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, half> frag_c_VKQ;
+
+ constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel.
+ constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy.
+ static_assert(VKQ_ratio <= nwarps, "VKQ_ratio must be <= nwarps.");
+
+ // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts:
+ constexpr int D_padded = D + 8;
+ constexpr int kqs_padded = FATTN_KQ_STRIDE + 8;
+ constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half);
+
+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
+ const float * Q_f = (const float *) (Q + nb02* blockIdx.y + nb01*ic0);
+ const half * K_h = (const half *) (K + nb12*(blockIdx.y / gqa_ratio));
+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape
+ const half * maskh = (const half *) mask + (nb31/sizeof(half))* ic0;
+ const half2 * mask2 = (const half2 *) mask + (nb31/sizeof(half))*(ic0/2);
+
+ const int stride_Q = nb01 / sizeof(float);
+ const int stride_KV = nb11 / sizeof(half);
+
+ frag_b Q_b[D/16][ncols/frag_n];
+
+ // A single buffer for temporarily holding tiles of KQ and VKQ parts:
+ constexpr int mem_KQ = ncols*kqs_padded*kqar;
+ constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded;
+ __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts];
+ float * KQ_f = (float *) KQ;
+ half2 * KQ2 = (half2 *) KQ;
+
+ float KQ_rowsum_f[ncols/nwarps] = {0.0f};
+ float KQ_max_f[ncols/nwarps];
+ float KQ_max_scale_f[ncols/nwarps] = {0.0f};
+
+#pragma unroll
+ for (int j = 0; j < ncols/nwarps; ++j) {
+ KQ_max_f[j] = -FLT_MAX/2.0f;
+ }
+
+ half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}};
+ half2 KQ_max_h2[ncols/nwarps];
+ half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}};
+
+#pragma unroll
+ for (int j = 0; j < ncols/nwarps; ++j) {
+ KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF);
+ }
+
+ __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice.
+ half2 * VKQ2 = (half2 *) VKQ;
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {
+ break;
+ }
+ VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f);
+ }
+ }
+
+ // Convert Q to half and apply scale, temporarily store in KQ:
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D && i >= D) {
+ break;
+ }
+ KQ[j*D_padded + i] = ic0 + j < ne01 ? Q_f[j*stride_Q + i] * scale : 0.0f;
+ }
+ }
+
+ __syncthreads();
+
+ // Load Q into tensor core fragments/registers since it will be used frequently:
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += 16) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+ nvcuda::wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ + j0*D_padded + i0, D_padded);
+ }
+ }
+
+ __syncthreads();
+
+ // Iterate over ne11 == previous tokens:
+ for (int k_VKQ_0 = ip*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE) {
+ // Calculate tile of KQ:
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) {
+ frag_c_KQ KQ_c[ncols/frag_n];
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::fill_fragment(KQ_c[j], 0.0f);
+ }
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) {
+ frag_a_K K_a;
+ nvcuda::wmma::load_matrix_sync(K_a, K_h + (k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV);
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]);
+ }
+ }
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+ nvcuda::wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, nvcuda::wmma::mem_col_major);
+ }
+ }
+
+ __syncthreads();
+
+ // Calculate softmax for each KQ column using the current max. value.
+ // The divisor is stored in KQ_rowsum and will be applied at the end.
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (std::is_same<KQ_acc_t, float>::value) {
+ float KQ_f_tmp[FATTN_KQ_STRIDE / WARP_SIZE];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ_f_tmp[k0/WARP_SIZE] = KQ_f[j*kqs_padded + k];
+ }
+
+ float KQ_max_new = KQ_max_f[j0/nwarps];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ_f_tmp[k0/WARP_SIZE] += mask ? __half2float(maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f;
+ KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/WARP_SIZE]);
+ }
+ KQ_max_new = warp_reduce_max(KQ_max_new);
+
+ const float diff = KQ_max_f[j0/nwarps] - KQ_max_new;
+ KQ_max_scale_f[j0/nwarps] = expf(diff);
+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {
+ KQ_max_scale_f[j0/nwarps] = 0.0f;
+ }
+ KQ_max_f[j0/nwarps] = KQ_max_new;
+
+ float KQ_rowsum_add = 0.0f;
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ const float diff = KQ_f_tmp[k0/WARP_SIZE] - KQ_max_f[j0/nwarps];
+ KQ_f_tmp[k0/WARP_SIZE] = expf(diff);
+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {
+ KQ_f_tmp[k0/WARP_SIZE] = 0.0f;
+ }
+ KQ_rowsum_add += KQ_f_tmp[k0/WARP_SIZE];
+ KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/WARP_SIZE];
+ }
+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);
+
+ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:
+ KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add;
+ } else {
+ half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*WARP_SIZE)];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ2_tmp[k0/WARP_SIZE] = KQ2[j*(kqs_padded/2) + k];
+ }
+
+ half2 KQ_max_new = KQ_max_h2[j0/nwarps];
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ KQ2_tmp[k0/WARP_SIZE] += mask ? mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f);
+ KQ_max_new = __hmax2(KQ_max_new, KQ2_tmp[k0/WARP_SIZE]);
+ }
+ KQ_max_new = __half2half2(warp_reduce_max(__hmax(__low2half(KQ_max_new), __high2half(KQ_max_new))));
+ const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new;
+ KQ_max_scale_h2[j0/nwarps] = h2exp(diff);
+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));
+ *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask;
+ KQ_max_h2[j0/nwarps] = KQ_max_new;
+
+ half2 KQ_rowsum_add = make_half2(0.0f, 0.0f);
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {
+ const int k = k0 + threadIdx.x;
+
+ const half2 diff = KQ2_tmp[k0/WARP_SIZE] - KQ_max_h2[j0/nwarps];
+ KQ2_tmp[k0/WARP_SIZE] = h2exp(diff);
+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));
+ *((uint32_t *) &KQ2_tmp[k0/WARP_SIZE]) &= ftz_mask;
+ KQ_rowsum_add += KQ2_tmp[k0/WARP_SIZE];
+ KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/WARP_SIZE];
+ }
+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);
+
+ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:
+ KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add;
+ }
+ }
+
+ __syncthreads();
+
+ frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n];
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {
+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;
+ nvcuda::wmma::load_matrix_sync(
+ KQ_b[k0/(VKQ_ratio*16)][j0/frag_n],
+ KQ + j0*(kqar*kqs_padded) + k,
+ kqar*kqs_padded);
+ }
+ }
+
+ frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n];
+#pragma unroll
+ for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) {
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], 0.0f);
+ }
+
+#pragma unroll
+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {
+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;
+
+ frag_a_V v_a;
+ nvcuda::wmma::load_matrix_sync(v_a, V_h + (k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV);
+#pragma unroll
+ for (int j = 0; j < ncols/frag_n; ++j) {
+ nvcuda::wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]);
+ }
+ }
+ }
+
+ __syncthreads();
+
+ const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded);
+#pragma unroll
+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) {
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {
+ nvcuda::wmma::store_matrix_sync(
+ KQ + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio),
+ VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n],
+ D_padded, nvcuda::wmma::mem_col_major);
+ }
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ half2 VKQ_scale;
+ if (std::is_same<KQ_acc_t, float>::value) {
+ VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]);
+ } else {
+ VKQ_scale = KQ_max_scale_h2[j0/nwarps];
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {
+ break;
+ }
+
+ half2 VKQ_add = make_half2(0.0f, 0.0f);
+#pragma unroll
+ for (int l = 0; l < VKQ_ratio; ++l) {
+ VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i];
+ }
+ VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add;
+ }
+ }
+
+ __syncthreads();
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {
+ const int j_VKQ = j0 + threadIdx.y;
+ if (ic0 + j_VKQ >= ne01) {
+ return;
+ }
+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;
+
+ float KQ_rowsum_j;
+ if (std::is_same<KQ_acc_t, float>::value) {
+ KQ_rowsum_j = KQ_rowsum_f[j0/nwarps];
+ } else {
+ KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]);
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {
+ const int i = i0 + threadIdx.x;
+ if (i0 + WARP_SIZE > D && i >= D) {
+ break;
+ }
+ float dst_val = VKQ[j_VKQ*D_padded + i];
+ if (parallel_blocks == 1) {
+ dst_val /= KQ_rowsum_j;
+ }
+ dst[j_dst*gridDim.y*D + blockIdx.y*D + i] = dst_val;
+ }
+
+ if (parallel_blocks == 1 || threadIdx.x != 0) {
+ continue;
+ }
+
+ float2 dst_meta_val;
+ if (std::is_same<KQ_acc_t, float>::value) {
+ dst_meta_val.x = KQ_max_f[j0/nwarps];
+ } else {
+ dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]);
+ }
+ dst_meta_val.y = KQ_rowsum_j;
+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = dst_meta_val;
+ }
+#else
+ NO_DEVICE_CODE;
+#endif // FP16_MMA_AVAILABLE
+}
+
+template<int D, int parallel_blocks> // D == head size
+__launch_bounds__(D, 1)
+static __global__ void flash_attn_combine_results(
+ const float * __restrict__ VKQ_parts,
+ const float2 * __restrict__ VKQ_meta,
+ float * __restrict__ dst) {
+#if FP16_AVAILABLE
+ VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x;
+ VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x;
+ dst += D * gridDim.y*blockIdx.x;
+
+ const int tid = threadIdx.x;
+ __builtin_assume(tid < D);
+
+ __shared__ float2 meta[parallel_blocks];
+ if (tid < 2*parallel_blocks) {
+ ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid];
+ }
+
+ __syncthreads();
+
+ float kqmax = meta[0].x;
+#pragma unroll
+ for (int l = 1; l < parallel_blocks; ++l) {
+ kqmax = max(kqmax, meta[l].x);
+ }
+
+ float VKQ_numerator = 0.0f;
+ float VKQ_denominator = 0.0f;
+#pragma unroll
+ for (int l = 0; l < parallel_blocks; ++l) {
+ const float diff = meta[l].x - kqmax;
+ const float KQ_max_scale = expf(diff);
+ const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD);
+ *((uint32_t *) &KQ_max_scale) &= ftz_mask;
+
+ VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid];
+ VKQ_denominator += KQ_max_scale * meta[l].y;
+ }
+
+ dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator;
+#else
+ NO_DEVICE_CODE;
+#endif // FP16_AVAILABLE
+}
+
+constexpr int get_max_power_of_2(int x) {
+ return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1;
+}
+
+static_assert(get_max_power_of_2(1) == 1, "Test failed.");
+static_assert(get_max_power_of_2(2) == 2, "Test failed.");
+static_assert(get_max_power_of_2(4) == 4, "Test failed.");
+static_assert(get_max_power_of_2(6) == 2, "Test failed.");
+
+// Number of VKQ rows calculated in parallel:
+constexpr int get_VKQ_stride(int D, int nwarps, int frag_m) {
+ return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m;
+}
+
+static_assert(get_VKQ_stride(128, 1, 32) == 32, "Test failed.");
+static_assert(get_VKQ_stride(128, 2, 32) == 64, "Test failed.");
+static_assert(get_VKQ_stride(128, 4, 32) == 128, "Test failed.");
+static_assert(get_VKQ_stride( 64, 1, 32) == 32, "Test failed.");
+static_assert(get_VKQ_stride( 64, 2, 32) == 64, "Test failed.");
+static_assert(get_VKQ_stride( 64, 4, 32) == 64, "Test failed.");
+static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed.");
+static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed.");
+static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed.");
+
+template <int D, int parallel_blocks> void launch_fattn_vec_f16(
+ const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,
+ ggml_cuda_pool & pool, cudaStream_t main_stream
+) {
+ ggml_cuda_pool_alloc<float> dst_tmp(pool);
+ ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);
+
+ if (parallel_blocks > 1) {
+ dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));
+ dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));
+ }
+
+ constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE;
+ const dim3 block_dim(WARP_SIZE, nwarps, 1);
+ const dim3 blocks_num(parallel_blocks*Q->ne[1], Q->ne[2], Q->ne[3]);
+ const int shmem = 0;
+
+ float scale;
+ memcpy(&scale, KQV->op_params, sizeof(float));
+
+ flash_attn_vec_ext_f16<D, parallel_blocks>
+ <<<blocks_num, block_dim, shmem, main_stream>>> (
+ (const char *) Q->data,
+ (const char *) K->data,
+ (const char *) V->data,
+ mask ? ((const char *) mask->data) : nullptr,
+ parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,
+ scale,
+ Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],
+ K->ne[0], K->ne[1], K->ne[2], K->ne[3],
+ mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,
+ Q->nb[1], Q->nb[2], Q->nb[3],
+ K->nb[1], K->nb[2], K->nb[3],
+ KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]
+ );
+ CUDA_CHECK(cudaGetLastError());
+
+ if (parallel_blocks == 1) {
+ return;
+ }
+
+ const dim3 block_dim_combine(D, 1, 1);
+ const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);
+ const int shmem_combine = 0;
+
+ flash_attn_combine_results<D, parallel_blocks>
+ <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>
+ (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);
+ CUDA_CHECK(cudaGetLastError());
+}
+
+template <int D, int cols_per_block, int nwarps, int parallel_blocks, typename KQ_acc_t> void launch_fattn_f16_impl(
+ const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,
+ ggml_cuda_pool & pool, cudaStream_t main_stream
+) {
+ ggml_cuda_pool_alloc<float> dst_tmp(pool);
+ ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);
+
+ if (parallel_blocks > 1) {
+ dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));
+ dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));
+ }
+
+ constexpr int frag_m = (cols_per_block) == 8 && (D) % 32 == 0 ? 32 : 16;
+ const dim3 block_dim(WARP_SIZE, nwarps, 1);
+ const dim3 blocks_num(parallel_blocks*(Q->ne[1] + cols_per_block - 1) / cols_per_block, Q->ne[2], Q->ne[3]);
+ const int shmem = 0;
+
+ float scale;
+ memcpy(&scale, KQV->op_params, sizeof(float));
+
+ flash_attn_ext_f16<D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t>
+ <<<blocks_num, block_dim, shmem, main_stream>>> (
+ (const char *) Q->data,
+ (const char *) K->data,
+ (const char *) V->data,
+ mask ? ((const char *) mask->data) : nullptr,
+ (parallel_blocks) == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,
+ scale,
+ Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],
+ K->ne[0], K->ne[1], K->ne[2], K->ne[3],
+ mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,
+ Q->nb[1], Q->nb[2], Q->nb[3],
+ K->nb[1], K->nb[2], K->nb[3],
+ KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]
+ );
+ CUDA_CHECK(cudaGetLastError());
+
+ if ((parallel_blocks) == 1) {
+ return;
+ }
+
+ const dim3 block_dim_combine(D, 1, 1);
+ const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);
+ const int shmem_combine = 0;
+
+ flash_attn_combine_results<D, parallel_blocks>
+ <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>
+ (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);
+ CUDA_CHECK(cudaGetLastError());
+}
+
+template <int D, int cols_per_block, int nwarps, typename KQ_acc_t> void launch_fattn_f16(
+ const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,
+ const int nsm, ggml_cuda_pool & pool, cudaStream_t main_stream
+) {
+ const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3];
+
+ if (4*blocks_num_pb1 < 2*nsm) {
+ launch_fattn_f16_impl<D, cols_per_block, nwarps, 4, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);
+ return;
+ }
+ if (2*blocks_num_pb1 < 2*nsm) {
+ launch_fattn_f16_impl<D, cols_per_block, nwarps, 2, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);
+ return;
+ }
+ launch_fattn_f16_impl<D, cols_per_block, nwarps, 1, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);
+}
+
+void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * Q = dst->src[0];
+ const ggml_tensor * K = dst->src[1];
+ const ggml_tensor * V = dst->src[2];
+
+ const ggml_tensor * mask = dst->src[3];
+
+ ggml_tensor * KQV = dst;
+
+ GGML_ASSERT(Q->type == GGML_TYPE_F32);
+ GGML_ASSERT(K->type == GGML_TYPE_F16);
+ GGML_ASSERT(V->type == GGML_TYPE_F16);
+ GGML_ASSERT(KQV->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);
+ GGML_ASSERT(!mask || mask->ne[1] >= GGML_PAD(Q->ne[1], 16) &&
+ "the Flash-Attention CUDA kernel requires the mask to be padded to 16 and at least n_queries big");
+
+ GGML_ASSERT(K->ne[1] % FATTN_KQ_STRIDE == 0 && "Incorrect KV cache padding.");
+
+ ggml_cuda_set_device(ctx.device);
+
+ const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
+
+ const int32_t precision = KQV->op_params[1];
+
+ if (precision != GGML_PREC_DEFAULT) {
+ if (Q->ne[1] <= 32 || Q->ne[0] > 128) {
+ constexpr int cols_per_block = 16;
+ constexpr int nwarps = 4;
+ switch (Q->ne[0]) {
+ case 64:
+ launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 80:
+ launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 96:
+ launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 112:
+ launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 128:
+ launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 256:
+ launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ default:
+ GGML_ASSERT(false);
+ break;
+ }
+ } else {
+ constexpr int cols_per_block = 32;
+ constexpr int nwarps = 4;
+ switch (Q->ne[0]) {
+ case 64:
+ launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 80:
+ launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 96:
+ launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 112:
+ launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 128:
+ launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ // case 256:
+ // launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ // break;
+ default:
+ GGML_ASSERT(false);
+ break;
+ }
+ }
+ return;
+ }
+
+ if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0) {
+ constexpr int parallel_blocks = 4;
+ switch (Q->ne[0]) {
+ case 64:
+ launch_fattn_vec_f16< 64, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());
+ break;
+ case 128:
+ launch_fattn_vec_f16<128, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());
+ break;
+ case 256:
+ launch_fattn_vec_f16<256, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());
+ break;
+ default:
+ GGML_ASSERT(false);
+ break;
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 8 && Q->ne[0] % WARP_SIZE == 0) {
+ constexpr int cols_per_block = 8;
+ constexpr int nwarps = 4;
+ switch (Q->ne[0]) {
+ case 64:
+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 96:
+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 128:
+ launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 256:
+ launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ default:
+ GGML_ASSERT(false);
+ break;
+ }
+ return;
+ }
+
+ if (Q->ne[1] <= 32) {
+ constexpr int cols_per_block = 16;
+ constexpr int nwarps = 4;
+ switch (Q->ne[0]) {
+ case 64:
+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 80:
+ launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 96:
+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 112:
+ launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 128:
+ launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 256:
+ launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ default:
+ GGML_ASSERT(false);
+ break;
+ }
+ return;
+ }
+
+ constexpr int cols_per_block = 32;
+ constexpr int nwarps = 4;
+ switch (Q->ne[0]) {
+ case 64:
+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 80:
+ launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 96:
+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 112:
+ launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 128:
+ launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ case 256:
+ launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());
+ break;
+ default:
+ GGML_ASSERT(false);
+ break;
+ }
+ return;
+}
+
template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
static __global__ void k_get_rows(
const void * src0, const int32_t * src1, dst_t * dst,
@@ -7924,13 +8878,14 @@ void ggml_cuda_op_mul_mat_vec_q(
GGML_UNUSED(src1_padded_row_size);
}
+template <int block_size>
static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {
- const int row = blockIdx.x;
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
const int tid = threadIdx.x;
float2 mean_var = make_float2(0.f, 0.f);
- for (int col = tid; col < ncols; col += WARP_SIZE) {
+ for (int col = tid; col < ncols; col += block_size) {
const float xi = x[row*ncols + col];
mean_var.x += xi;
mean_var.y += xi * xi;
@@ -7938,16 +8893,28 @@ static __global__ void norm_f32(const float * x, float * dst, const int ncols, c
// sum up partial sums
mean_var = warp_reduce_sum(mean_var);
+ if (block_size > WARP_SIZE) {
+ __shared__ float2 s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = mean_var;
+ }
+ __syncthreads();
+ mean_var = s_sum[lane_id];
+ mean_var = warp_reduce_sum(mean_var);
+ }
const float mean = mean_var.x / ncols;
const float var = mean_var.y / ncols - mean * mean;
const float inv_std = rsqrtf(var + eps);
- for (int col = tid; col < ncols; col += WARP_SIZE) {
+ for (int col = tid; col < ncols; col += block_size) {
dst[row*ncols + col] = (x[row*ncols + col] - mean) * inv_std;
}
}
+template <int block_size>
static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {
// blockIdx.x: num_groups idx
// threadIdx.x: block_size idx
@@ -7962,62 +8929,117 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr
float tmp = 0.0f; // partial sum for thread in warp
- for (int j = start; j < end; j += WARP_SIZE) {
+ for (int j = start; j < end; j += block_size) {
tmp += x[j];
}
tmp = warp_reduce_sum(tmp);
+ if (block_size > WARP_SIZE) {
+ __shared__ float s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = tmp;
+ }
+ __syncthreads();
+ tmp = s_sum[lane_id];
+ tmp = warp_reduce_sum(tmp);
+ }
float mean = tmp / group_size;
tmp = 0.0f;
- for (int j = start; j < end; j += WARP_SIZE) {
+ for (int j = start; j < end; j += block_size) {
float xi = x[j] - mean;
dst[j] = xi;
tmp += xi * xi;
}
tmp = warp_reduce_sum(tmp);
+ if (block_size > WARP_SIZE) {
+ __shared__ float s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = tmp;
+ }
+ __syncthreads();
+ tmp = s_sum[lane_id];
+ tmp = warp_reduce_sum(tmp);
+ }
float variance = tmp / group_size;
float scale = rsqrtf(variance + eps);
- for (int j = start; j < end; j += WARP_SIZE) {
+ for (int j = start; j < end; j += block_size) {
dst[j] *= scale;
}
}
-static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps, int nrows) {
- const int row = blockIdx.x;
+template <int block_size>
+static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {
+ const int row = blockIdx.x*blockDim.y + threadIdx.y;
const int tid = threadIdx.x;
float tmp = 0.0f; // partial sum for thread in warp
- for (int col = tid; col < ncols; col += WARP_SIZE) {
+ for (int col = tid; col < ncols; col += block_size) {
const float xi = x[row*ncols + col];
tmp += xi * xi;
}
- const float mean = warp_reduce_sum(tmp) / ncols;
+ // sum up partial sums
+ tmp = warp_reduce_sum(tmp);
+ if (block_size > WARP_SIZE) {
+ __shared__ float s_sum[32];
+ int warp_id = threadIdx.x / WARP_SIZE;
+ int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ s_sum[warp_id] = tmp;
+ }
+ __syncthreads();
+ tmp = s_sum[lane_id];
+ tmp = warp_reduce_sum(tmp);
+ }
+
+ const float mean = tmp / ncols;
const float scale = rsqrtf(mean + eps);
- for (int col = tid; col < ncols; col += WARP_SIZE) {
+ for (int col = tid; col < ncols; col += block_size) {
dst[row*ncols + col] = scale * x[row*ncols + col];
}
}
static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
GGML_ASSERT(ncols % WARP_SIZE == 0);
- norm_f32<<<nrows, WARP_SIZE, 0, stream>>>(x, dst, ncols, eps);
+ if (ncols < 1024) {
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ } else {
+ const dim3 block_dims(1024, 1, 1);
+ norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ }
}
static void group_norm_f32_cuda(const float * x, float * dst, const int num_groups, const int group_size, const int ne_elements, cudaStream_t stream) {
static const float eps = 1e-6f;
- group_norm_f32<<<num_groups, WARP_SIZE, 0, stream>>>(x, dst, group_size, ne_elements, eps);
+ if (group_size < 1024) {
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
+ } else {
+ const dim3 block_dims(1024, 1, 1);
+ group_norm_f32<1024><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
+ }
}
static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
GGML_ASSERT(ncols % WARP_SIZE == 0);
- rms_norm_f32<<<nrows, WARP_SIZE, 0, stream>>>(x, dst, ncols, eps, nrows);
+ if (ncols < 1024) {
+ const dim3 block_dims(WARP_SIZE, 1, 1);
+ rms_norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ } else {
+ const dim3 block_dims(1024, 1, 1);
+ rms_norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
+ }
}
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
@@ -8593,8 +9615,18 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
CUDA_CHECK(cudaGetLastError());
}
-template <bool vals_smem, int ncols_template, int block_size_template>
-static __global__ void soft_max_f32(const float * x, const float * mask, const float * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {
+template <typename T>
+static __device__ __forceinline__ float t2f32(T val) {
+ return (float) val;
+}
+
+template <>
+__device__ float __forceinline__ t2f32<half>(half val) {
+ return __half2float(val);
+}
+
+template <bool vals_smem, int ncols_template, int block_size_template, typename T>
+static __global__ void soft_max_f32(const float * x, const T * mask, const T * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {
const int ncols = ncols_template == 0 ? ncols_par : ncols_template;
const int tid = threadIdx.x;
@@ -8621,7 +9653,7 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const f
extern __shared__ float data_soft_max_f32[];
float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication
// shared memory buffer to cache values between iterations:
- float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + rowx*ncols;
+ float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + (int64_t)rowx*ncols;
float max_val = -INFINITY;
@@ -8633,10 +9665,10 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const f
break;
}
- const int ix = rowx*ncols + col;
- const int iy = rowy*ncols + col;
+ const int64_t ix = (int64_t)rowx*ncols + col;
+ const int64_t iy = (int64_t)rowy*ncols + col;
- const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + (pos ? slope*pos[col] : 0.0f);
+ const float val = x[ix]*scale + (mask ? t2f32(mask[iy]) : 0.0f) + (pos ? slope*t2f32(pos[col]) : 0.0f);
vals[col] = val;
max_val = max(max_val, val);
@@ -8702,12 +9734,13 @@ static __global__ void soft_max_f32(const float * x, const float * mask, const f
return;
}
- const int idst = rowx*ncols + col;
+ const int64_t idst = (int64_t)rowx*ncols + col;
dst[idst] = vals[col] * inv_sum;
}
}
-static void soft_max_f32_cuda(const float * x, const float * mask, const float * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {
+template<typename T>
+static void soft_max_f32_cuda(const float * x, const T * mask, const T * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {
int nth = WARP_SIZE;
while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;
const dim3 block_dims(nth, 1, 1);
@@ -8760,15 +9793,19 @@ static void soft_max_f32_cuda(const float * x, const float * mask, const float *
void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
+ const ggml_tensor * src2 = dst->src[2];
+
const float * src0_d = (const float *)src0->data;
- const float * src1_d = src1 ? (const float *)src1->data : nullptr;
+ const void * src1_d = src1 ? (const void *)src1->data : nullptr;
+
float * dst_d = (float *)dst->data;
cudaStream_t stream = ctx.stream();
GGML_ASSERT(src0->type == GGML_TYPE_F32);
GGML_ASSERT( dst->type == GGML_TYPE_F32);
- GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional
+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional
+ GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F16 || src2->type == GGML_TYPE_F32); // src2 contains positions and it is optional
const int64_t ne00 = src0->ne[0];
const int64_t nrows_x = ggml_nrows(src0);
@@ -8781,16 +9818,27 @@ void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));
// positions tensor
- float * src2_dd = nullptr;
+ void * src2_d = nullptr;
- ggml_tensor * src2 = dst->src[2];
const bool use_src2 = src2 != nullptr;
if (use_src2) {
- src2_dd = (float *)src2->data;
+ src2_d = (void *)src2->data;
}
- soft_max_f32_cuda(src0_d, src1_d, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);
+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);
+
+ if (use_f16) {
+ const half * src1_dd = (const half *)src1_d;
+ const half * src2_dd = (const half *)src2_d;
+
+ soft_max_f32_cuda(src0_d, src1_dd, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);
+ } else {
+ const float * src1_dd = (const float *)src1_d;
+ const float * src2_dd = (const float *)src2_d;
+
+ soft_max_f32_cuda(src0_d, src1_dd, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);
+ }
}
static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {
@@ -9167,23 +10215,6 @@ void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
#include "ggml.h"
#include "ggml-backend-impl.h"
-
-#include <algorithm>
-#include <array>
-#include <atomic>
-#include <cinttypes>
-#include <cstddef>
-#include <cstdint>
-#include <float.h>
-#include <limits>
-#include <map>
-#include <memory>
-#include <mutex>
-#include <stdint.h>
-#include <stdio.h>
-#include <string>
-#include <vector>
-
static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
[[noreturn]]
@@ -9284,6 +10315,7 @@ static ggml_cuda_device_info ggml_cuda_init() {
info.devices[id].cc = 100*prop.major + 10*prop.minor;
#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
info.devices[id].smpb = prop.sharedMemPerBlock;
+ info.devices[id].nsm = prop.multiProcessorCount;
}
for (int id = 0; id < info.device_count; ++id) {
@@ -11434,6 +12466,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
case GGML_OP_ARGSORT:
ggml_cuda_op_argsort(ctx, dst);
break;
+ case GGML_OP_FLASH_ATTN_EXT:
+ ggml_cuda_flash_attn_ext(ctx, dst);
+ break;
default:
return false;
}
@@ -11708,6 +12743,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons
case GGML_OP_ARANGE:
case GGML_OP_TIMESTEP_EMBEDDING:
case GGML_OP_LEAKY_RELU:
+ case GGML_OP_FLASH_ATTN_EXT:
return true;
default:
return false;
diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.h
index ee24f60b..90870cb4 100644
--- a/llama.cpp/ggml-impl.h
+++ b/llama.cpp/ggml-impl.h
@@ -315,7 +315,7 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b)
#endif // defined(__ARM_NEON)
-#if defined(__ARM_NEON) && !defined(__MSC_VER)
+#if defined(__ARM_NEON) && !defined(_MSC_VER)
#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.m
index ed5408a4..d32007a7 100644
--- a/llama.cpp/ggml-metal.m
+++ b/llama.cpp/ggml-metal.m
@@ -29,6 +29,7 @@ static const struct ggml_backend_api *g_backend;
#define ggml_type_size g_backend->ggml_type_size
#define ggml_guid_matches g_backend->ggml_guid_matches
#define ggml_is_empty g_backend->ggml_is_empty
+#define ggml_are_same_shape g_backend->ggml_are_same_shape
void ggml_metal_link(const struct ggml_backend_api *backend_api) {
g_backend = backend_api;
@@ -73,8 +74,10 @@ enum ggml_metal_kernel_type {
GGML_METAL_KERNEL_TYPE_GELU_QUICK_4,
GGML_METAL_KERNEL_TYPE_SILU,
GGML_METAL_KERNEL_TYPE_SILU_4,
- GGML_METAL_KERNEL_TYPE_SOFT_MAX,
- GGML_METAL_KERNEL_TYPE_SOFT_MAX_4,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32,
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4,
GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF,
GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8,
GGML_METAL_KERNEL_TYPE_GET_ROWS_F32,
@@ -204,6 +207,14 @@ enum ggml_metal_kernel_type {
GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,
GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC,
GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128,
+ GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256,
GGML_METAL_KERNEL_TYPE_CPY_F32_F16,
GGML_METAL_KERNEL_TYPE_CPY_F32_F32,
GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0,
@@ -472,7 +483,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {
}
/*
- GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \
+ GGML_METAL_LOG_INFO("%s: loaded %-40s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \
(int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \
(int) kernel->pipeline.threadExecutionWidth); \
*/
@@ -488,172 +499,182 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {
return NULL; \
} \
} else { \
- GGML_METAL_LOG_WARN("%s: skipping %-32s (not supported)\n", __func__, "kernel_"#name); \
+ GGML_METAL_LOG_WARN("%s: skipping %-40s (not supported)\n", __func__, "kernel_"#name); \
}
// simd_sum and simd_max requires MTLGPUFamilyApple7
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CLAMP, clamp, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_4, gelu_4, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK_4, gelu_quick_4, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU_4, silu_4, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX, soft_max, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_4, soft_max_4, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M, get_rows_iq1_m, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32, mul_mv_iq1_m_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, mul_mv_iq4_xs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, ctx->support_simdgroup_reduction);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32, mul_mm_id_iq1_m_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CLAMP, clamp, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_4, gelu_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK_4, gelu_quick_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU_4, silu_4, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16, soft_max_f16, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4, soft_max_f16_4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32, soft_max_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4, soft_max_f32_4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M, get_rows_iq1_m, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32, mul_mv_iq1_m_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, mul_mv_iq4_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, ctx->support_simdgroup_reduction);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32, mul_mm_id_iq1_m_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64, flash_attn_ext_f16_h64, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80, flash_attn_ext_f16_h80, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96, flash_attn_ext_f16_h96, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112, flash_attn_ext_f16_h112, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128, flash_attn_ext_f16_h128, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, flash_attn_ext_f16_h256, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128, flash_attn_ext_vec_f16_h128, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, flash_attn_ext_vec_f16_h256, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);
}
[metal_library release];
@@ -772,6 +793,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const
case GGML_OP_TIMESTEP_EMBEDDING:
case GGML_OP_ARGSORT:
case GGML_OP_LEAKY_RELU:
+ case GGML_OP_FLASH_ATTN_EXT:
return true;
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
@@ -1355,20 +1377,33 @@ static enum ggml_status ggml_metal_graph_compute(
} break;
case GGML_OP_SOFT_MAX:
{
+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);
+ GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F16 || src2->type == GGML_TYPE_F32);
+
int nth = 32; // SIMD width
id<MTLComputePipelineState> pipeline = nil;
+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);
+
if (ne00%4 == 0) {
while (nth < ne00/4 && nth < 256) {
nth *= 2;
}
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_4].pipeline;
+ if (use_f16) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4].pipeline;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline;
+ }
} else {
while (nth < ne00 && nth < 1024) {
nth *= 2;
}
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline;
+ if (use_f16) {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16].pipeline;
+ } else {
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32].pipeline;
+ }
}
float scale;
@@ -2532,6 +2567,161 @@ static enum ggml_status ggml_metal_graph_compute(
[encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
} break;
+ case GGML_OP_FLASH_ATTN_EXT:
+ {
+ GGML_ASSERT(ne00 % 4 == 0);
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+
+ struct ggml_tensor * src3 = gf->nodes[i]->src[3];
+
+ GGML_ASSERT(ggml_are_same_shape(src1, src2));
+ GGML_ASSERT(src3);
+
+ size_t offs_src3 = 0;
+
+ id<MTLBuffer> id_src3 = src3 ? ggml_metal_get_buffer(src3, &offs_src3) : nil;
+
+ GGML_ASSERT(!src3 || src3->type == GGML_TYPE_F16);
+ GGML_ASSERT(!src3 || src3->ne[1] >= GGML_PAD(src0->ne[1], 8) &&
+ "the Flash-Attention Metal kernel requires the mask to be padded to 8 and at least n_queries big");
+
+ const int64_t ne30 = src3 ? src3->ne[0] : 0; GGML_UNUSED(ne30);
+ const int64_t ne31 = src3 ? src3->ne[1] : 0;
+ const int64_t ne32 = src3 ? src3->ne[2] : 0; GGML_UNUSED(ne32);
+ const int64_t ne33 = src3 ? src3->ne[3] : 0; GGML_UNUSED(ne33);
+
+ const uint64_t nb30 = src3 ? src3->nb[0] : 0; GGML_UNUSED(nb30);
+ const uint64_t nb31 = src3 ? src3->nb[1] : 0;
+ const uint64_t nb32 = src3 ? src3->nb[2] : 0; GGML_UNUSED(nb32);
+ const uint64_t nb33 = src3 ? src3->nb[3] : 0; GGML_UNUSED(nb33);
+
+ const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);
+
+ float scale;
+ memcpy(&scale, dst->op_params, sizeof(float));
+
+ id<MTLComputePipelineState> pipeline = nil;
+
+ bool use_vec_kernel = false;
+
+ if (ne01 >= 4 || (ne00%128 != 0)) {
+ switch (ne00) {
+ case 64: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64 ].pipeline; break;
+ case 80: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80 ].pipeline; break;
+ case 96: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96 ].pipeline; break;
+ case 112: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112].pipeline; break;
+ case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128].pipeline; break;
+ case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256].pipeline; break;
+ default:
+ {
+ GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);
+ GGML_METAL_LOG_ERROR("add template specialization for this size\n");
+ GGML_ASSERT(false && "add template specialization for this size");
+ }
+ }
+ } else {
+ use_vec_kernel = true;
+
+ switch (ne00) {
+ case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128].pipeline; break;
+ case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256].pipeline; break;
+ default:
+ {
+ GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);
+ GGML_METAL_LOG_ERROR("add template specialization for this size\n");
+ GGML_ASSERT(false && "add template specialization for this size");
+ }
+ }
+ }
+
+ [encoder setComputePipelineState:pipeline];
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];
+ [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3];
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:4];
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:5];
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:6];
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:7];
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:8];
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:9];
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:10];
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:11];
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:12];
+ [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:13];
+ [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:14];
+ [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:15];
+ [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:16];
+ [encoder setBytes:&nb10 length:sizeof(uint64_t) atIndex:17];
+ [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:18];
+ [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:19];
+ [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:20];
+ [encoder setBytes:&ne31 length:sizeof( int64_t) atIndex:21];
+ [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:22];
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:23];
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:24];
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:25];
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:26];
+ [encoder setBytes:&scale length:sizeof( float) atIndex:27];
+
+ if (!use_vec_kernel) {
+ // half8x8 kernel
+ const int64_t nqptg = 8; // queries per threadgroup !! sync with kernel template arguments !!
+ const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!
+
+ GGML_ASSERT(nqptg <= 32);
+ GGML_ASSERT(nqptg % 8 == 0);
+ GGML_ASSERT(ncpsg % 32 == 0);
+
+ int64_t nsgmax = 2;
+
+ while (true) {
+ const size_t smem = nqptg*(ne00 + 2*nsgmax*(ncpsg + nqptg))*(sizeof(float)/2);
+ if (smem > ctx->device.maxThreadgroupMemoryLength) {
+ break;
+ }
+ nsgmax *= 2;
+ }
+ nsgmax /= 2;
+
+ // simdgroups per threadgroup (a.k.a. warps)
+ const int64_t nsg = ne01 <= nqptg ? MAX(4, MIN(nsgmax, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32))) : 4;
+
+ const size_t smem = nqptg*(ne00 + 2*nsg*(ncpsg + nqptg))*(sizeof(float)/2);
+
+ //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);
+ GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
+
+ [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
+ } else {
+ // half1x4 kernel
+ const int64_t nqptg = 1; // queries per threadgroup !! sync with kernel template arguments !!
+ const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!
+
+ GGML_ASSERT(nqptg <= 32);
+ GGML_ASSERT(nqptg % 1 == 0);
+ GGML_ASSERT(ncpsg % 32 == 0);
+
+ // simdgroups per threadgroup (a.k.a. warps)
+ const int64_t nsgt = MAX(2, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32));
+
+ int64_t nsg = 1;
+ while (nsg <= nsgt) {
+ nsg *= 2;
+ }
+ nsg /= 2;
+
+ const size_t smem = (nqptg*(ne00 + 2*nsg*(ncpsg + nqptg)) + nsg*ne00)*(sizeof(float)/2);
+
+ //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);
+ GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
+ [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];
+
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
+ }
+ } break;
case GGML_OP_DUP:
case GGML_OP_CPY:
case GGML_OP_CONT:
@@ -2619,6 +2809,11 @@ static enum ggml_status ggml_metal_graph_compute(
MTLCommandBufferStatus status = [command_buffer status];
if (status != MTLCommandBufferStatusCompleted) {
GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);
+ if (status == MTLCommandBufferStatusError) {
+ NSString * error_code = [command_buffer error].localizedDescription;
+ GGML_METAL_LOG_INFO("error: %s\n", [error_code UTF8String]);
+ }
+
return GGML_STATUS_FAILED;
}
}
@@ -2735,10 +2930,13 @@ GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backe
UNUSED(buft);
}
-static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device) {
+static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device, size_t size_aligned) {
+#ifndef GGML_METAL_NDEBUG
#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
if (@available(macOS 10.12, iOS 16.0, *)) {
- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",
+ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f / %8.2f)",
+ __func__,
+ size_aligned / 1024.0 / 1024.0,
device.currentAllocatedSize / 1024.0 / 1024.0,
device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
@@ -2748,10 +2946,15 @@ static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device) {
GGML_METAL_LOG_INFO("\n");
}
} else {
- GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);
+ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f)\n",
+ __func__,
+ size_aligned / 1024.0 / 1024.0,
+ device.currentAllocatedSize / 1024.0 / 1024.0);
}
+#endif
#endif
UNUSED(device);
+ UNUSED(size_aligned);
}
GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
@@ -2785,8 +2988,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buff
return NULL;
}
- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);
- ggml_backend_metal_log_allocated_size(device);
+ //ggml_backend_metal_log_allocated_size(device, size_aligned);
return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);
}
@@ -2873,7 +3075,7 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,
return false;
}
- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);
+ ggml_backend_metal_log_allocated_size(device, size_aligned);
++ctx->n_buffers;
} else {
@@ -2896,7 +3098,8 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,
return false;
}
- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, offs = %12ld", __func__, size_step_aligned / 1024.0 / 1024.0, i);
+ ggml_backend_metal_log_allocated_size(device, size_step_aligned);
+
if (i + size_step < size) {
GGML_METAL_LOG_INFO("\n");
}
@@ -2905,8 +3108,6 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,
}
}
- ggml_backend_metal_log_allocated_size(device);
-
return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);
}
diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metal
index e666ea56..382e4278 100644
--- a/llama.cpp/ggml-metal.metal
+++ b/llama.cpp/ggml-metal.metal
@@ -2206,11 +2206,12 @@ kernel void kernel_sum_rows(
dst_row[0] = row_sum;
}
+template<typename T>
kernel void kernel_soft_max(
- device const float * src0,
- device const float * src1,
- device const float * src2,
- device float * dst,
+ device const char * src0,
+ device const char * src1,
+ device const char * src2,
+ device char * dst,
constant int64_t & ne00,
constant int64_t & ne01,
constant int64_t & ne02,
@@ -2229,10 +2230,10 @@ kernel void kernel_soft_max(
const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
- device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
- device const float * pmask = src1 != src0 ? src1 + i01*ne00 : nullptr;
- device const float * ppos = src2 != src0 ? src2 : nullptr;
- device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
+ device const float * psrc0 = (device const float *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
+ device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00 : nullptr;
+ device const T * ppos = src2 != src0 ? (device const T *) src2 : nullptr;
+ device float * pdst = (device float *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
float slope = 0.0f;
@@ -2310,11 +2311,12 @@ kernel void kernel_soft_max(
}
}
+template<typename T>
kernel void kernel_soft_max_4(
- device const float * src0,
- device const float * src1,
- device const float * src2,
- device float * dst,
+ device const char * src0,
+ device const char * src1,
+ device const char * src2,
+ device char * dst,
constant int64_t & ne00,
constant int64_t & ne01,
constant int64_t & ne02,
@@ -2333,10 +2335,10 @@ kernel void kernel_soft_max_4(
const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
- device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
- device const float4 * pmask = src1 != src0 ? (device const float4 *)(src1 + i01*ne00) : nullptr;
- device const float4 * ppos = src2 != src0 ? (device const float4 *)(src2) : nullptr;
- device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
+ device const float4 * psrc4 = (device const float4 *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;
+ device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00/4 : nullptr;
+ device const T * ppos = src2 != src0 ? (device const T *) src2 : nullptr;
+ device float4 * pdst4 = (device float4 *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;
float slope = 0.0f;
@@ -2353,7 +2355,7 @@ kernel void kernel_soft_max_4(
float4 lmax4 = -INFINITY;
for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
- lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));
+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (float4)((pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)));
}
const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));
@@ -2379,7 +2381,7 @@ kernel void kernel_soft_max_4(
// parallel sum
float4 lsum4 = 0.0f;
for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
- const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);
+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (float4)((pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f))) - max_val);
lsum4 += exp_psrc4;
pdst4[i00] = exp_psrc4;
}
@@ -2416,6 +2418,14 @@ kernel void kernel_soft_max_4(
}
}
+typedef decltype(kernel_soft_max<float>) kernel_soft_max_t;
+typedef decltype(kernel_soft_max_4<float4>) kernel_soft_max_4_t;
+
+template [[host_name("kernel_soft_max_f16")]] kernel kernel_soft_max_t kernel_soft_max<half>;
+template [[host_name("kernel_soft_max_f32")]] kernel kernel_soft_max_t kernel_soft_max<float>;
+template [[host_name("kernel_soft_max_f16_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4<half4>;
+template [[host_name("kernel_soft_max_f32_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4<float4>;
+
kernel void kernel_diag_mask_inf(
device const float * src0,
device float * dst,
@@ -3938,6 +3948,632 @@ kernel void kernel_leaky_relu_f32(
dst[tpig] = src0[tpig] > 0.0f ? src0[tpig] : src0[tpig] * slope;
}
+typedef void (flash_attn_ext_f16_t)(
+ device const char * q,
+ device const char * k,
+ device const char * v,
+ device const char * mask,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne31,
+ constant uint64_t & nb31,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant float & scale,
+ threadgroup half * shared,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]);
+
+// ref: https://arxiv.org/pdf/2307.08691.pdf
+template<int64_t D, int64_t Q = 8, int64_t C = 32> // head size, queries per threadgroup, cache items per threadgroup
+kernel void kernel_flash_attn_ext_f16(
+ device const char * q,
+ device const char * k,
+ device const char * v,
+ device const char * mask,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne31,
+ constant uint64_t & nb31,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant float & scale,
+ threadgroup half * shared [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+ const short nsg = ntg.y; // number of simdgroups
+
+ const short iq3 = tgpig[2];
+ const short iq2 = tgpig[1];
+ const short iq1 = tgpig[0]*Q;
+
+ const short D4 = D/4;
+ const short D8 = D/8;
+ const short Q8 = Q/8;
+ const short NW = N_SIMDWIDTH;
+ const short SH = (C + Q); // shared memory per simdgroup in (half)
+
+ const short T = D + 2*nsg*SH; // shared memory size per query in (half)
+ const short TF = T/2; // shared memory size per query in (float)
+ const short T4 = T/4; // shared memory size per query in (half4)
+
+ threadgroup half * sq = (threadgroup half *) (shared + 0*D); // holds the query data
+ threadgroup half4 * sq4 = (threadgroup half4 *) (shared + 0*D); // same as above but in half4
+ threadgroup float * ss = (threadgroup float *) (shared + 2*sgitg*SH + 1*D); // scratch buffer for attention and diagonal matrix
+
+ // store the result for all queries in local memory in 8x8 matrices (the O matrix from the paper)
+ simdgroup_half8x8 lo[D8];
+
+ // load heads from Q to shared memory
+ for (short j = sgitg; j < Q; j += nsg) {
+ device const float4 * q4 = (device const float4 *) ((device const char *) q + ((iq1 + j)*nb01 + iq2*nb02 + iq3*nb03));
+
+ for (short i = tiisg; i < D4; i += NW) {
+ if (iq1 + j < ne01) {
+ sq4[j*T4 + i] = (half4) q4[i];
+ } else {
+ sq4[j*T4 + i] = 0.0h;
+ }
+ }
+ }
+
+ // zero out lo
+ for (short i = 0; i < D8; ++i) {
+ lo[i] = make_filled_simdgroup_matrix<half, 8>(0.0h);
+ }
+
+ // zero out shared memory SH
+ for (short j = 0; j < Q; ++j) {
+ for (short i = tiisg; i < SH; i += NW) {
+ ss[j*TF + i] = 0.0f;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ {
+ float S[Q] = { [0 ... Q-1] = 0.0h };
+ float M[Q] = { [0 ... Q-1] = -FLT_MAX/2 };
+
+ // assume K and V are same shape
+ const short ne22 = ne12;
+ const short ne23 = ne13;
+
+ const uint nb21 = nb11;
+ const uint nb22 = nb12;
+ const uint nb23 = nb13;
+
+ // broadcast
+ const short rk2 = ne02/ne12;
+ const short rk3 = ne03/ne13;
+
+ const short rv2 = ne02/ne22;
+ const short rv3 = ne03/ne23;
+
+ // k indices
+ const short ik2 = iq2/rk2;
+ const short ik3 = iq3/rk3;
+
+ // v indices
+ const short iv2 = iq2/rv2;
+ const short iv3 = iq3/rv3;
+
+ // load the queries from shared memory into local memory
+ simdgroup_half8x8 mq[D8];
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_load(mq[i], sq + i*8, T);
+ }
+
+ // pointer to the mask
+ device const half * mp = (device const half *) (mask + iq1*nb31);
+
+ // prepare diagonal scale matrix
+ simdgroup_float8x8 mscale(scale);
+
+ // loop over the KV cache
+ // each simdgroup handles blocks of Q rows and C columns
+ for (int ic0 = 0; ic0 < ne11; ic0 += C*nsg) {
+ const int ic = ic0 + C*sgitg;
+ if (ic >= ne11) {
+ break;
+ }
+
+ // Q*K^T
+ {
+ for (short cc = 0; cc < C/8; ++cc) {
+ simdgroup_float8x8 mqk = make_filled_simdgroup_matrix<float, 8>(0.h);
+
+ device const half * pk = (device const half *) ((device const char *) k + ((ic + 8*cc)*nb11 + ik2*nb12 + ik3*nb13));
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_half8x8 mk;
+ simdgroup_load(mk, pk + i*8, nb11/sizeof(half), 0, true); // transpose
+
+ simdgroup_multiply_accumulate(mqk, mq[i], mk, mqk);
+ }
+
+ // mqk = mqk*scale + mask
+ simdgroup_half8x8 mm;
+ simdgroup_load(mm, mp + ic + 8*cc, nb31/sizeof(half), 0, false);
+ simdgroup_multiply_accumulate(mqk, mqk, mscale, mm);
+
+ simdgroup_store(mqk, ss + 8*cc, TF, 0, false);
+ }
+ }
+
+ // used to detect blocks full of -INF
+ float smax = -INFINITY;
+
+ // online softmax
+ {
+ float ms[Q];
+
+ for (short j = 0; j < Q; ++j) {
+ const short p = tiisg;
+
+ const float m = M[j];
+ const float s = ss[j*TF + p];
+
+ smax = simd_max(max(smax, s));
+ M[j] = simd_max(max(M[j], s));
+
+ ms[j] = exp(m - M[j]);
+ const float vs = exp(s - M[j]);
+
+ S[j] = S[j]*ms[j] + simd_sum(vs);
+
+ // the P matrix from the paper (Q rows, C columns)
+ ss[j*TF + p] = vs;
+ }
+
+ // create a QxQ diagonal matrix for rescaling the output
+ if (tiisg < Q) {
+ ss[tiisg*TF + C + tiisg] = ms[tiisg];
+ }
+ }
+
+ // skip -INF blocks
+ if (smax == -INFINITY) {
+ continue;
+ }
+
+ // O = diag(ms)*O
+ {
+ simdgroup_float8x8 mm;
+ simdgroup_load(mm, ss + C, TF, 0, false);
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_multiply(lo[i], mm, lo[i]);
+ }
+ }
+
+ // O = O + (Q*K^T)*V
+ {
+ for (short cc = 0; cc < C/8; ++cc) {
+ device const half * pv = (device const half *) ((device const char *) v + ((ic + 8*cc)*nb21 + iv2*nb22 + iv3*nb23));
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_half8x8 mk;
+ simdgroup_load(mk, pv + i*8, nb21/sizeof(half), 0, false);
+
+ simdgroup_float8x8 mv;
+ simdgroup_load(mv, ss + 8*cc, TF, 0, false);
+
+ simdgroup_multiply_accumulate(lo[i], mv, mk, lo[i]);
+ }
+ }
+ }
+ }
+
+ // these are needed for reducing the results from the simdgroups (reuse the ss buffer)
+ for (short j = 0; j < Q; ++j) {
+ if (tiisg == 0) {
+ ss[j*TF + 0] = S[j];
+ ss[j*TF + 1] = M[j];
+ }
+ }
+ }
+
+ // reduce the warps sequentially
+ for (short sg = 1; sg < nsg; ++sg) {
+ float S = { 0.0h };
+ float M = { -FLT_MAX/2 };
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // each simdgroup stores its output to shared memory, reusing sq
+ if (sgitg == sg) {
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_store(lo[i], sq + i*8, T, 0, false);
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // the first simdgroup accumulates the results from the other simdgroups
+ if (sgitg == 0) {
+ for (short j = 0; j < Q; ++j) {
+ const float S0 = ss[j*TF + 0];
+ const float S1 = ss[j*TF + sg*SH + 0];
+
+ const float M0 = ss[j*TF + 1];
+ const float M1 = ss[j*TF + sg*SH + 1];
+
+ M = max(M0, M1);
+
+ const float ms0 = exp(M0 - M);
+ const float ms1 = exp(M1 - M);
+
+ S = S0*ms0 + S1*ms1;
+
+ if (tiisg == 0) {
+ ss[j*TF + 0] = S;
+ ss[j*TF + 1] = M;
+
+ ss[j*TF + C + j ] = ms0;
+ ss[j*TF + C + j + sg*SH] = ms1;
+ }
+ }
+
+ // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1
+ {
+ simdgroup_half8x8 t;
+ simdgroup_float8x8 ms0;
+ simdgroup_float8x8 ms1;
+
+ simdgroup_load(ms0, ss + C, TF, 0, false);
+ simdgroup_load(ms1, ss + C + sg*SH, TF, 0, false);
+
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_load (t, sq + i*8, T, 0, false);
+ simdgroup_multiply(t, ms1, t);
+
+ simdgroup_multiply_accumulate(lo[i], ms0, lo[i], t);
+ }
+ }
+ }
+ }
+
+ // store result to shared memory (reuse sq)
+ if (sgitg == 0) {
+ for (short i = 0; i < D8; ++i) {
+ simdgroup_store(lo[i], sq + i*8, T, 0, false);
+ }
+ }
+
+ device float4 * dst4 = (device float4 *) dst;
+
+ // final rescale with 1/S and store to global memory
+ if (sgitg == 0) {
+ for (short j = 0; j < Q && iq1 + j < ne01; ++j) {
+ const float S = ss[j*TF + 0];
+
+ for (short i = tiisg; i < D4; i += NW) {
+ dst4[(iq3*ne2*ne1 + iq2 + (iq1 + j)*ne1)*D4 + i] = (float4) sq4[j*T4 + i]/S;
+ }
+ }
+ }
+}
+
+template [[host_name("kernel_flash_attn_ext_f16_h64" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<64>;
+template [[host_name("kernel_flash_attn_ext_f16_h80" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<80>;
+template [[host_name("kernel_flash_attn_ext_f16_h96" )]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<96>;
+template [[host_name("kernel_flash_attn_ext_f16_h112")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<112>;
+template [[host_name("kernel_flash_attn_ext_f16_h128")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<128>;
+template [[host_name("kernel_flash_attn_ext_f16_h256")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_f16<256>;
+
+template<int64_t D, int64_t Q = 1, int64_t C = 32> // head size, queries per threadgroup, cache items per threadgroup
+kernel void kernel_flash_attn_ext_vec_f16(
+ device const char * q,
+ device const char * k,
+ device const char * v,
+ device const char * mask,
+ device float * dst,
+ constant int64_t & ne00,
+ constant int64_t & ne01,
+ constant int64_t & ne02,
+ constant int64_t & ne03,
+ constant uint64_t & nb00,
+ constant uint64_t & nb01,
+ constant uint64_t & nb02,
+ constant uint64_t & nb03,
+ constant int64_t & ne10,
+ constant int64_t & ne11,
+ constant int64_t & ne12,
+ constant int64_t & ne13,
+ constant uint64_t & nb10,
+ constant uint64_t & nb11,
+ constant uint64_t & nb12,
+ constant uint64_t & nb13,
+ constant int64_t & ne31,
+ constant uint64_t & nb31,
+ constant int64_t & ne0,
+ constant int64_t & ne1,
+ constant int64_t & ne2,
+ constant int64_t & ne3,
+ constant float & scale,
+ threadgroup half * shared [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tpitg[[thread_position_in_threadgroup]],
+ uint3 ntg[[threads_per_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+ const short nsg = ntg.y; // number of simdgroups
+
+ const short iq3 = tgpig[2];
+ const short iq2 = tgpig[1];
+ const short iq1 = tgpig[0];
+
+ const short D4 = D/4;
+ const short NW = N_SIMDWIDTH;
+ const short SH = (C + Q); // shared memory per simdgroup in (half)
+
+ const short T = D + 2*nsg*SH; // shared memory size per query in (half)
+
+ //threadgroup half * sq = (threadgroup half *) (shared + 0*D); // holds the query data
+ threadgroup half4 * sq4 = (threadgroup half4 *) (shared + 0*D); // same as above but in half4
+ threadgroup float * ss = (threadgroup float *) (shared + 2*sgitg*SH + 1*D); // scratch buffer for attention and diagonal matrix
+ threadgroup float4 * ss4 = (threadgroup float4 *) (shared + 2*sgitg*SH + 1*D); // same as above but in half4
+ threadgroup half4 * sr4 = (threadgroup half4 *) (shared + sgitg*D + 1*T); // scratch buffer for the results
+
+ // store the result for all queries in local memory in 8x8 matrices (the O matrix from the paper)
+ half4 lo[D4/NW];
+
+ // load heads from Q to shared memory
+ device const float4 * q4 = (device const float4 *) ((device const char *) q + (iq1*nb01 + iq2*nb02 + iq3*nb03));
+
+ for (short i = tiisg; i < D4; i += NW) {
+ if (iq1 < ne01) {
+ sq4[i] = (half4) q4[i];
+ } else {
+ sq4[i] = 0.0h;
+ }
+ }
+
+ // zero out lo
+ for (short i = tiisg; i < D4; i += NW) {
+ lo[i/NW] = 0.0h;
+ }
+
+ // zero out shared memory SH
+ for (short i = tiisg; i < SH/4; i += NW) {
+ ss4[i] = 0.0h;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ {
+ float S = { 0.0h };
+ float M = { -FLT_MAX/2 };
+
+ // assume K and V are same shape
+ const short ne22 = ne12;
+ const short ne23 = ne13;
+
+ const uint nb21 = nb11;
+ const uint nb22 = nb12;
+ const uint nb23 = nb13;
+
+ // broadcast
+ const short rk2 = ne02/ne12;
+ const short rk3 = ne03/ne13;
+
+ const short rv2 = ne02/ne22;
+ const short rv3 = ne03/ne23;
+
+ // k indices
+ const short ik2 = iq2 / rk2;
+ const short ik3 = iq3 / rk3;
+
+ // v indices
+ const short iv2 = iq2 / rv2;
+ const short iv3 = iq3 / rv3;
+
+ // load the queries from shared memory into local memory
+ half4 mq[D4];
+
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ mq[i] = sq4[i];
+ }
+
+ // pointer to the mask
+ device const half4 * mp4 = (device const half4 *) (mask + iq1*nb31);
+
+ // loop over the KV cache
+ // each simdgroup handles blocks of Q rows and C columns
+ for (int ic0 = 0; ic0 < ne11; ic0 += C*nsg) {
+ const int ic = ic0 + C*sgitg;
+ if (ic >= ne11) {
+ break;
+ }
+
+ // Q*K^T
+ {
+#pragma unroll
+ for (short cc = 0; cc < C/4; ++cc) {
+ float4 mqk = { 0.0h };
+
+ device const half4 * pk4 = (device const half4 *) ((device const char *) k + ((ic + 4*cc)*nb11 + ik2*nb12 + ik3*nb13));
+
+#pragma unroll
+ for (short ii = 0; ii < D4; ii += NW) {
+ const short i = ii + tiisg;
+
+ half4x4 mk;
+ mk[0] = pk4[i + 0*(nb11/8)];
+ mk[1] = pk4[i + 1*(nb11/8)];
+ mk[2] = pk4[i + 2*(nb11/8)];
+ mk[3] = pk4[i + 3*(nb11/8)];
+
+ mqk += (float4) (mq[i] * mk);
+ }
+
+ // reduce the results from the threads in the simdgroup
+ mqk += simd_shuffle_down(mqk, 16);
+ mqk += simd_shuffle_down(mqk, 8);
+ mqk += simd_shuffle_down(mqk, 4);
+ mqk += simd_shuffle_down(mqk, 2);
+ mqk += simd_shuffle_down(mqk, 1);
+
+ // mqk = mqk*scale + mask
+ if (tiisg == 0) {
+ float4 mm = (float4) mp4[ic/4 + cc];
+ mqk = mqk*scale + mm;
+
+ ss4[cc] = mqk;
+ }
+ }
+ }
+
+ // online softmax
+ {
+ const short p = tiisg;
+
+ const float m = M;
+ const float s = ss[p];
+
+ M = simd_max(max(M, s));
+
+ const float ms = exp(m - M);
+ const float vs = exp(s - M);
+
+ S = S*ms + simd_sum(vs);
+
+ // the P matrix from the paper (Q rows, C columns)
+ ss[p] = vs;
+
+ // O = diag(ms)*O
+#pragma unroll
+ for (short ii = 0; ii < D4; ii += NW) {
+ const short i = ii + tiisg;
+ lo[i/NW] *= ms;
+ }
+ }
+
+ // O = O + (Q*K^T)*V
+ {
+#pragma unroll
+ for (short cc = 0; cc < C/4; ++cc) {
+ device const half4 * pv4 = (device const half4 *) ((device const char *) v + ((ic + 4*cc)*nb21 + iv2*nb22 + iv3*nb23));
+
+#pragma unroll
+ for (short ii = 0; ii < D4; ii += NW) {
+ const short i = ii + tiisg;
+
+ lo[i/NW] += pv4[i + 0*(nb21/8)] * ss[4*cc + 0];
+ lo[i/NW] += pv4[i + 1*(nb21/8)] * ss[4*cc + 1];
+ lo[i/NW] += pv4[i + 2*(nb21/8)] * ss[4*cc + 2];
+ lo[i/NW] += pv4[i + 3*(nb21/8)] * ss[4*cc + 3];
+ }
+ }
+ }
+
+ }
+
+ // these are needed for reducing the results from the simdgroups (reuse the ss buffer)
+ if (tiisg == 0) {
+ ss[0] = S;
+ ss[1] = M;
+ }
+ }
+
+ // store results to shared memory
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ sr4[i] = lo[ii/NW];
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // parallel reduce
+ for (short r = nsg/2; r > 0; r >>= 1) {
+ if (sgitg < r) {
+ const float S0 = ss[ 0];
+ const float S1 = ss[r*SH + 0];
+
+ const float M0 = ss[ 1];
+ const float M1 = ss[r*SH + 1];
+
+ const float M = max(M0, M1);
+
+ const float ms0 = exp(M0 - M);
+ const float ms1 = exp(M1 - M);
+
+ const float S = S0*ms0 + S1*ms1;
+
+ if (tiisg == 0) {
+ ss[0] = S;
+ ss[1] = M;
+ }
+
+ // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ sr4[i] = sr4[i]*ms0 + sr4[i + r*D4]*ms1;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ device float4 * dst4 = (device float4 *) dst;
+
+ // final rescale with 1/S and store to global memory
+ if (sgitg == 0) {
+ const float S = ss[0];
+
+ for (short ii = 0; ii < D4; ii += NW) {
+ short i = ii + tiisg;
+ dst4[(iq3*ne2*ne1 + iq2 + (iq1)*ne1)*D4 + i] = (float4) sr4[i]/S;
+ }
+ }
+}
+
+template [[host_name("kernel_flash_attn_ext_vec_f16_h128")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_vec_f16<128>;
+template [[host_name("kernel_flash_attn_ext_vec_f16_h256")]] kernel flash_attn_ext_f16_t kernel_flash_attn_ext_vec_f16<256>;
+
kernel void kernel_cpy_f16_f16(
device const half * src0,
device half * dst,
diff --git a/llama.cpp/ggml-quants-amd-avx.c b/llama.cpp/ggml-quants-amd-avx.c
new file mode 100644
index 00000000..dc6ace8f
--- /dev/null
+++ b/llama.cpp/ggml-quants-amd-avx.c
@@ -0,0 +1,100 @@
+#ifdef __x86_64__
+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_amd_avx
+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_amd_avx
+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_amd_avx
+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_amd_avx
+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_amd_avx
+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_amd_avx
+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_amd_avx
+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_amd_avx
+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_amd_avx
+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_amd_avx
+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_amd_avx
+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_amd_avx
+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_amd_avx
+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_amd_avx
+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_amd_avx
+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_amd_avx
+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_amd_avx
+#define quantize_row_q4_0 quantize_row_q4_0_amd_avx
+#define quantize_row_q4_1 quantize_row_q4_1_amd_avx
+#define quantize_row_q5_0 quantize_row_q5_0_amd_avx
+#define quantize_row_q5_1 quantize_row_q5_1_amd_avx
+#define quantize_row_q8_0 quantize_row_q8_0_amd_avx
+#define quantize_row_q8_1 quantize_row_q8_1_amd_avx
+#define quantize_row_q2_K quantize_row_q2_K_amd_avx
+#define quantize_row_q3_K quantize_row_q3_K_amd_avx
+#define quantize_row_q4_K quantize_row_q4_K_amd_avx
+#define quantize_row_q5_K quantize_row_q5_K_amd_avx
+#define quantize_row_q6_K quantize_row_q6_K_amd_avx
+#define quantize_row_q8_K quantize_row_q8_K_amd_avx
+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_amd_avx
+#define quantize_row_iq4_nl quantize_row_iq4_nl_amd_avx
+#define quantize_row_iq4_xs quantize_row_iq4_xs_amd_avx
+#define quantize_row_iq3_s quantize_row_iq3_s_amd_avx
+#define quantize_row_iq2_s quantize_row_iq2_s_amd_avx
+#define dequantize_row_q4_0 dequantize_row_q4_0_amd_avx
+#define dequantize_row_q4_1 dequantize_row_q4_1_amd_avx
+#define dequantize_row_q5_0 dequantize_row_q5_0_amd_avx
+#define dequantize_row_q5_1 dequantize_row_q5_1_amd_avx
+#define dequantize_row_q8_0 dequantize_row_q8_0_amd_avx
+#define dequantize_row_q2_K dequantize_row_q2_K_amd_avx
+#define dequantize_row_q3_K dequantize_row_q3_K_amd_avx
+#define dequantize_row_q4_K dequantize_row_q4_K_amd_avx
+#define dequantize_row_q5_K dequantize_row_q5_K_amd_avx
+#define dequantize_row_q6_K dequantize_row_q6_K_amd_avx
+#define dequantize_row_q8_K dequantize_row_q8_K_amd_avx
+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_amd_avx
+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_amd_avx
+#define dequantize_row_iq2_s dequantize_row_iq2_s_amd_avx
+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_amd_avx
+#define dequantize_row_iq1_s dequantize_row_iq1_s_amd_avx
+#define dequantize_row_iq1_m dequantize_row_iq1_m_amd_avx
+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_amd_avx
+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_amd_avx
+#define dequantize_row_iq3_s dequantize_row_iq3_s_amd_avx
+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_amd_avx
+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_amd_avx
+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_amd_avx
+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_amd_avx
+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_amd_avx
+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_amd_avx
+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_amd_avx
+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_amd_avx
+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_amd_avx
+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_amd_avx
+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_amd_avx
+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_amd_avx
+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_amd_avx
+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_amd_avx
+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_amd_avx
+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_amd_avx
+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_amd_avx
+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_amd_avx
+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_amd_avx
+#define quantize_iq2_xxs quantize_iq2_xxs_amd_avx
+#define quantize_iq2_xs quantize_iq2_xs_amd_avx
+#define quantize_iq2_s quantize_iq2_s_amd_avx
+#define quantize_iq3_xxs quantize_iq3_xxs_amd_avx
+#define quantize_iq1_s quantize_iq1_s_amd_avx
+#define quantize_iq1_m quantize_iq1_m_amd_avx
+#define quantize_iq4_nl quantize_iq4_nl_amd_avx
+#define quantize_iq4_xs quantize_iq4_xs_amd_avx
+#define quantize_iq3_s quantize_iq3_s_amd_avx
+#define quantize_q2_K quantize_q2_K_amd_avx
+#define quantize_q3_K quantize_q3_K_amd_avx
+#define quantize_q4_K quantize_q4_K_amd_avx
+#define quantize_q5_K quantize_q5_K_amd_avx
+#define quantize_q6_K quantize_q6_K_amd_avx
+#define quantize_q4_0 quantize_q4_0_amd_avx
+#define quantize_q4_1 quantize_q4_1_amd_avx
+#define quantize_q5_0 quantize_q5_0_amd_avx
+#define quantize_q5_1 quantize_q5_1_amd_avx
+#define quantize_q8_0 quantize_q8_0_amd_avx
+#define iq2xs_init_impl iq2xs_init_impl_amd_avx
+#define iq2xs_free_impl iq2xs_free_impl_amd_avx
+#define iq3xs_init_impl iq3xs_init_impl_amd_avx
+#define iq3xs_free_impl iq3xs_free_impl_amd_avx
+#define ggml_validate_row_data ggml_validate_row_data_amd_avx
+#include "ggml-quants.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-quants-amd-avx2.c b/llama.cpp/ggml-quants-amd-avx2.c
new file mode 100644
index 00000000..3a12b6f4
--- /dev/null
+++ b/llama.cpp/ggml-quants-amd-avx2.c
@@ -0,0 +1,100 @@
+#ifdef __x86_64__
+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_amd_avx2
+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_amd_avx2
+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_amd_avx2
+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_amd_avx2
+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_amd_avx2
+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_amd_avx2
+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_amd_avx2
+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_amd_avx2
+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_amd_avx2
+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_amd_avx2
+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_amd_avx2
+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_amd_avx2
+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_amd_avx2
+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_amd_avx2
+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_amd_avx2
+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_amd_avx2
+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_amd_avx2
+#define quantize_row_q4_0 quantize_row_q4_0_amd_avx2
+#define quantize_row_q4_1 quantize_row_q4_1_amd_avx2
+#define quantize_row_q5_0 quantize_row_q5_0_amd_avx2
+#define quantize_row_q5_1 quantize_row_q5_1_amd_avx2
+#define quantize_row_q8_0 quantize_row_q8_0_amd_avx2
+#define quantize_row_q8_1 quantize_row_q8_1_amd_avx2
+#define quantize_row_q2_K quantize_row_q2_K_amd_avx2
+#define quantize_row_q3_K quantize_row_q3_K_amd_avx2
+#define quantize_row_q4_K quantize_row_q4_K_amd_avx2
+#define quantize_row_q5_K quantize_row_q5_K_amd_avx2
+#define quantize_row_q6_K quantize_row_q6_K_amd_avx2
+#define quantize_row_q8_K quantize_row_q8_K_amd_avx2
+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_amd_avx2
+#define quantize_row_iq4_nl quantize_row_iq4_nl_amd_avx2
+#define quantize_row_iq4_xs quantize_row_iq4_xs_amd_avx2
+#define quantize_row_iq3_s quantize_row_iq3_s_amd_avx2
+#define quantize_row_iq2_s quantize_row_iq2_s_amd_avx2
+#define dequantize_row_q4_0 dequantize_row_q4_0_amd_avx2
+#define dequantize_row_q4_1 dequantize_row_q4_1_amd_avx2
+#define dequantize_row_q5_0 dequantize_row_q5_0_amd_avx2
+#define dequantize_row_q5_1 dequantize_row_q5_1_amd_avx2
+#define dequantize_row_q8_0 dequantize_row_q8_0_amd_avx2
+#define dequantize_row_q2_K dequantize_row_q2_K_amd_avx2
+#define dequantize_row_q3_K dequantize_row_q3_K_amd_avx2
+#define dequantize_row_q4_K dequantize_row_q4_K_amd_avx2
+#define dequantize_row_q5_K dequantize_row_q5_K_amd_avx2
+#define dequantize_row_q6_K dequantize_row_q6_K_amd_avx2
+#define dequantize_row_q8_K dequantize_row_q8_K_amd_avx2
+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_amd_avx2
+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_amd_avx2
+#define dequantize_row_iq2_s dequantize_row_iq2_s_amd_avx2
+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_amd_avx2
+#define dequantize_row_iq1_s dequantize_row_iq1_s_amd_avx2
+#define dequantize_row_iq1_m dequantize_row_iq1_m_amd_avx2
+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_amd_avx2
+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_amd_avx2
+#define dequantize_row_iq3_s dequantize_row_iq3_s_amd_avx2
+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_amd_avx2
+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_amd_avx2
+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_amd_avx2
+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_amd_avx2
+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_amd_avx2
+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_amd_avx2
+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_amd_avx2
+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_amd_avx2
+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_amd_avx2
+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_amd_avx2
+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_amd_avx2
+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_amd_avx2
+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_amd_avx2
+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_amd_avx2
+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_amd_avx2
+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_amd_avx2
+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_amd_avx2
+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_amd_avx2
+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_amd_avx2
+#define quantize_iq2_xxs quantize_iq2_xxs_amd_avx2
+#define quantize_iq2_xs quantize_iq2_xs_amd_avx2
+#define quantize_iq2_s quantize_iq2_s_amd_avx2
+#define quantize_iq3_xxs quantize_iq3_xxs_amd_avx2
+#define quantize_iq1_s quantize_iq1_s_amd_avx2
+#define quantize_iq1_m quantize_iq1_m_amd_avx2
+#define quantize_iq4_nl quantize_iq4_nl_amd_avx2
+#define quantize_iq4_xs quantize_iq4_xs_amd_avx2
+#define quantize_iq3_s quantize_iq3_s_amd_avx2
+#define quantize_q2_K quantize_q2_K_amd_avx2
+#define quantize_q3_K quantize_q3_K_amd_avx2
+#define quantize_q4_K quantize_q4_K_amd_avx2
+#define quantize_q5_K quantize_q5_K_amd_avx2
+#define quantize_q6_K quantize_q6_K_amd_avx2
+#define quantize_q4_0 quantize_q4_0_amd_avx2
+#define quantize_q4_1 quantize_q4_1_amd_avx2
+#define quantize_q5_0 quantize_q5_0_amd_avx2
+#define quantize_q5_1 quantize_q5_1_amd_avx2
+#define quantize_q8_0 quantize_q8_0_amd_avx2
+#define iq2xs_init_impl iq2xs_init_impl_amd_avx2
+#define iq2xs_free_impl iq2xs_free_impl_amd_avx2
+#define iq3xs_init_impl iq3xs_init_impl_amd_avx2
+#define iq3xs_free_impl iq3xs_free_impl_amd_avx2
+#define ggml_validate_row_data ggml_validate_row_data_amd_avx2
+#include "ggml-quants.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-quants-amd-avx512.c b/llama.cpp/ggml-quants-amd-avx512.c
new file mode 100644
index 00000000..590eff07
--- /dev/null
+++ b/llama.cpp/ggml-quants-amd-avx512.c
@@ -0,0 +1,100 @@
+#ifdef __x86_64__
+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_amd_avx512
+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_amd_avx512
+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_amd_avx512
+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_amd_avx512
+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_amd_avx512
+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_amd_avx512
+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_amd_avx512
+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_amd_avx512
+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_amd_avx512
+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_amd_avx512
+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_amd_avx512
+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_amd_avx512
+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_amd_avx512
+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_amd_avx512
+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_amd_avx512
+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_amd_avx512
+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_amd_avx512
+#define quantize_row_q4_0 quantize_row_q4_0_amd_avx512
+#define quantize_row_q4_1 quantize_row_q4_1_amd_avx512
+#define quantize_row_q5_0 quantize_row_q5_0_amd_avx512
+#define quantize_row_q5_1 quantize_row_q5_1_amd_avx512
+#define quantize_row_q8_0 quantize_row_q8_0_amd_avx512
+#define quantize_row_q8_1 quantize_row_q8_1_amd_avx512
+#define quantize_row_q2_K quantize_row_q2_K_amd_avx512
+#define quantize_row_q3_K quantize_row_q3_K_amd_avx512
+#define quantize_row_q4_K quantize_row_q4_K_amd_avx512
+#define quantize_row_q5_K quantize_row_q5_K_amd_avx512
+#define quantize_row_q6_K quantize_row_q6_K_amd_avx512
+#define quantize_row_q8_K quantize_row_q8_K_amd_avx512
+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_amd_avx512
+#define quantize_row_iq4_nl quantize_row_iq4_nl_amd_avx512
+#define quantize_row_iq4_xs quantize_row_iq4_xs_amd_avx512
+#define quantize_row_iq3_s quantize_row_iq3_s_amd_avx512
+#define quantize_row_iq2_s quantize_row_iq2_s_amd_avx512
+#define dequantize_row_q4_0 dequantize_row_q4_0_amd_avx512
+#define dequantize_row_q4_1 dequantize_row_q4_1_amd_avx512
+#define dequantize_row_q5_0 dequantize_row_q5_0_amd_avx512
+#define dequantize_row_q5_1 dequantize_row_q5_1_amd_avx512
+#define dequantize_row_q8_0 dequantize_row_q8_0_amd_avx512
+#define dequantize_row_q2_K dequantize_row_q2_K_amd_avx512
+#define dequantize_row_q3_K dequantize_row_q3_K_amd_avx512
+#define dequantize_row_q4_K dequantize_row_q4_K_amd_avx512
+#define dequantize_row_q5_K dequantize_row_q5_K_amd_avx512
+#define dequantize_row_q6_K dequantize_row_q6_K_amd_avx512
+#define dequantize_row_q8_K dequantize_row_q8_K_amd_avx512
+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_amd_avx512
+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_amd_avx512
+#define dequantize_row_iq2_s dequantize_row_iq2_s_amd_avx512
+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_amd_avx512
+#define dequantize_row_iq1_s dequantize_row_iq1_s_amd_avx512
+#define dequantize_row_iq1_m dequantize_row_iq1_m_amd_avx512
+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_amd_avx512
+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_amd_avx512
+#define dequantize_row_iq3_s dequantize_row_iq3_s_amd_avx512
+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_amd_avx512
+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_amd_avx512
+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_amd_avx512
+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_amd_avx512
+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_amd_avx512
+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_amd_avx512
+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_amd_avx512
+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_amd_avx512
+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_amd_avx512
+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_amd_avx512
+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_amd_avx512
+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_amd_avx512
+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_amd_avx512
+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_amd_avx512
+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_amd_avx512
+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_amd_avx512
+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_amd_avx512
+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_amd_avx512
+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_amd_avx512
+#define quantize_iq2_xxs quantize_iq2_xxs_amd_avx512
+#define quantize_iq2_xs quantize_iq2_xs_amd_avx512
+#define quantize_iq2_s quantize_iq2_s_amd_avx512
+#define quantize_iq3_xxs quantize_iq3_xxs_amd_avx512
+#define quantize_iq1_s quantize_iq1_s_amd_avx512
+#define quantize_iq1_m quantize_iq1_m_amd_avx512
+#define quantize_iq4_nl quantize_iq4_nl_amd_avx512
+#define quantize_iq4_xs quantize_iq4_xs_amd_avx512
+#define quantize_iq3_s quantize_iq3_s_amd_avx512
+#define quantize_q2_K quantize_q2_K_amd_avx512
+#define quantize_q3_K quantize_q3_K_amd_avx512
+#define quantize_q4_K quantize_q4_K_amd_avx512
+#define quantize_q5_K quantize_q5_K_amd_avx512
+#define quantize_q6_K quantize_q6_K_amd_avx512
+#define quantize_q4_0 quantize_q4_0_amd_avx512
+#define quantize_q4_1 quantize_q4_1_amd_avx512
+#define quantize_q5_0 quantize_q5_0_amd_avx512
+#define quantize_q5_1 quantize_q5_1_amd_avx512
+#define quantize_q8_0 quantize_q8_0_amd_avx512
+#define iq2xs_init_impl iq2xs_init_impl_amd_avx512
+#define iq2xs_free_impl iq2xs_free_impl_amd_avx512
+#define iq3xs_init_impl iq3xs_init_impl_amd_avx512
+#define iq3xs_free_impl iq3xs_free_impl_amd_avx512
+#define ggml_validate_row_data ggml_validate_row_data_amd_avx512
+#include "ggml-quants.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-quants-arm80.c b/llama.cpp/ggml-quants-arm80.c
new file mode 100644
index 00000000..61783c01
--- /dev/null
+++ b/llama.cpp/ggml-quants-arm80.c
@@ -0,0 +1,100 @@
+#ifdef __aarch64__
+#define quantize_row_q4_0_reference quantize_row_q4_0_reference_arm80
+#define quantize_row_q4_1_reference quantize_row_q4_1_reference_arm80
+#define quantize_row_q5_0_reference quantize_row_q5_0_reference_arm80
+#define quantize_row_q5_1_reference quantize_row_q5_1_reference_arm80
+#define quantize_row_q8_0_reference quantize_row_q8_0_reference_arm80
+#define quantize_row_q8_1_reference quantize_row_q8_1_reference_arm80
+#define quantize_row_q2_K_reference quantize_row_q2_K_reference_arm80
+#define quantize_row_q3_K_reference quantize_row_q3_K_reference_arm80
+#define quantize_row_q4_K_reference quantize_row_q4_K_reference_arm80
+#define quantize_row_q5_K_reference quantize_row_q5_K_reference_arm80
+#define quantize_row_q6_K_reference quantize_row_q6_K_reference_arm80
+#define quantize_row_q8_K_reference quantize_row_q8_K_reference_arm80
+#define quantize_row_iq3_xxs_reference quantize_row_iq3_xxs_reference_arm80
+#define quantize_row_iq4_nl_reference quantize_row_iq4_nl_reference_arm80
+#define quantize_row_iq4_xs_reference quantize_row_iq4_xs_reference_arm80
+#define quantize_row_iq3_s_reference quantize_row_iq3_s_reference_arm80
+#define quantize_row_iq2_s_reference quantize_row_iq2_s_reference_arm80
+#define quantize_row_q4_0 quantize_row_q4_0_arm80
+#define quantize_row_q4_1 quantize_row_q4_1_arm80
+#define quantize_row_q5_0 quantize_row_q5_0_arm80
+#define quantize_row_q5_1 quantize_row_q5_1_arm80
+#define quantize_row_q8_0 quantize_row_q8_0_arm80
+#define quantize_row_q8_1 quantize_row_q8_1_arm80
+#define quantize_row_q2_K quantize_row_q2_K_arm80
+#define quantize_row_q3_K quantize_row_q3_K_arm80
+#define quantize_row_q4_K quantize_row_q4_K_arm80
+#define quantize_row_q5_K quantize_row_q5_K_arm80
+#define quantize_row_q6_K quantize_row_q6_K_arm80
+#define quantize_row_q8_K quantize_row_q8_K_arm80
+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_arm80
+#define quantize_row_iq4_nl quantize_row_iq4_nl_arm80
+#define quantize_row_iq4_xs quantize_row_iq4_xs_arm80
+#define quantize_row_iq3_s quantize_row_iq3_s_arm80
+#define quantize_row_iq2_s quantize_row_iq2_s_arm80
+#define dequantize_row_q4_0 dequantize_row_q4_0_arm80
+#define dequantize_row_q4_1 dequantize_row_q4_1_arm80
+#define dequantize_row_q5_0 dequantize_row_q5_0_arm80
+#define dequantize_row_q5_1 dequantize_row_q5_1_arm80
+#define dequantize_row_q8_0 dequantize_row_q8_0_arm80
+#define dequantize_row_q2_K dequantize_row_q2_K_arm80
+#define dequantize_row_q3_K dequantize_row_q3_K_arm80
+#define dequantize_row_q4_K dequantize_row_q4_K_arm80
+#define dequantize_row_q5_K dequantize_row_q5_K_arm80
+#define dequantize_row_q6_K dequantize_row_q6_K_arm80
+#define dequantize_row_q8_K dequantize_row_q8_K_arm80
+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_arm80
+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_arm80
+#define dequantize_row_iq2_s dequantize_row_iq2_s_arm80
+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_arm80
+#define dequantize_row_iq1_s dequantize_row_iq1_s_arm80
+#define dequantize_row_iq1_m dequantize_row_iq1_m_arm80
+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_arm80
+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_arm80
+#define dequantize_row_iq3_s dequantize_row_iq3_s_arm80
+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_arm80
+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_arm80
+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_arm80
+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_arm80
+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_arm80
+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_arm80
+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_arm80
+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_arm80
+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_arm80
+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_arm80
+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_arm80
+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_arm80
+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_arm80
+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_arm80
+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_arm80
+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_arm80
+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_arm80
+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_arm80
+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_arm80
+#define quantize_iq2_xxs quantize_iq2_xxs_arm80
+#define quantize_iq2_xs quantize_iq2_xs_arm80
+#define quantize_iq2_s quantize_iq2_s_arm80
+#define quantize_iq3_xxs quantize_iq3_xxs_arm80
+#define quantize_iq1_s quantize_iq1_s_arm80
+#define quantize_iq1_m quantize_iq1_m_arm80
+#define quantize_iq4_nl quantize_iq4_nl_arm80
+#define quantize_iq4_xs quantize_iq4_xs_arm80
+#define quantize_iq3_s quantize_iq3_s_arm80
+#define quantize_q2_K quantize_q2_K_arm80
+#define quantize_q3_K quantize_q3_K_arm80
+#define quantize_q4_K quantize_q4_K_arm80
+#define quantize_q5_K quantize_q5_K_arm80
+#define quantize_q6_K quantize_q6_K_arm80
+#define quantize_q4_0 quantize_q4_0_arm80
+#define quantize_q4_1 quantize_q4_1_arm80
+#define quantize_q5_0 quantize_q5_0_arm80
+#define quantize_q5_1 quantize_q5_1_arm80
+#define quantize_q8_0 quantize_q8_0_arm80
+#define iq2xs_init_impl iq2xs_init_impl_arm80
+#define iq2xs_free_impl iq2xs_free_impl_arm80
+#define iq3xs_init_impl iq3xs_init_impl_arm80
+#define iq3xs_free_impl iq3xs_free_impl_arm80
+#define ggml_validate_row_data ggml_validate_row_data_arm80
+#include "ggml-quants.inc"
+#endif // __aarch64__
diff --git a/llama.cpp/ggml-quants.cpp b/llama.cpp/ggml-quants.cpp
new file mode 100644
index 00000000..797d0990
--- /dev/null
+++ b/llama.cpp/ggml-quants.cpp
@@ -0,0 +1,1381 @@
+#include <cosmo.h>
+#include <sys/auxv.h>
+#include <libc/sysv/consts/hwcap.h>
+#include "ggml-quants.h"
+
+extern "C" void quantize_row_q4_0_reference_amd_avx512(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_0_reference_amd_avx2(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_0_reference_amd_avx(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_0_reference_arm80(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q4_1_reference_amd_avx512(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_reference_amd_avx2(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_reference_amd_avx(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_reference_arm80(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q5_0_reference_amd_avx512(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_reference_amd_avx2(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_reference_amd_avx(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_reference_arm80(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q5_1_reference_amd_avx512(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_reference_amd_avx2(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_reference_amd_avx(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_reference_arm80(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q8_0_reference_amd_avx512(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_reference_amd_avx2(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_reference_amd_avx(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_reference_arm80(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q8_1_reference_amd_avx512(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_reference_amd_avx2(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_reference_amd_avx(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_reference_arm80(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q2_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_reference_arm80(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q3_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_reference_arm80(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q4_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_reference_arm80(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q5_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_reference_arm80(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q6_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_reference_arm80(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q8_K_reference_amd_avx512(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_reference_amd_avx2(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_reference_amd_avx(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_reference_arm80(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq3_xxs_reference_amd_avx512(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_reference_amd_avx2(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_reference_amd_avx(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_reference_arm80(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq4_nl_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_reference_amd_avx (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_reference_arm80 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq4_xs_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_reference_amd_avx (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_reference_arm80 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq3_s_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_reference_amd_avx (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_reference_arm80 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq2_s_reference_amd_avx512 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_reference_amd_avx2 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_reference_amd_avx (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_reference_arm80 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q4_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q4_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q5_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q5_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q8_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q8_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q2_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q3_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q4_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q5_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q6_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_q8_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq3_xxs_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq4_nl_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq4_xs_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq3_s_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void quantize_row_iq2_s_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q4_0_amd_avx512(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_0_amd_avx2(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_0_amd_avx(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_0_arm80(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q4_1_amd_avx512(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_1_amd_avx2(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_1_amd_avx(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_1_arm80(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q5_0_amd_avx512(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_0_amd_avx2(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_0_amd_avx(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_0_arm80(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q5_1_amd_avx512(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_1_amd_avx2(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_1_amd_avx(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_1_arm80(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q8_0_amd_avx512(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_0_amd_avx2(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_0_amd_avx(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_0_arm80(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q2_K_amd_avx512(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q2_K_amd_avx2(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q2_K_amd_avx(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q2_K_arm80(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q3_K_amd_avx512(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q3_K_amd_avx2(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q3_K_amd_avx(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q3_K_arm80(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q4_K_amd_avx512(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_K_amd_avx2(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_K_amd_avx(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_K_arm80(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q5_K_amd_avx512(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_K_amd_avx2(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_K_amd_avx(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_K_arm80(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q6_K_amd_avx512(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q6_K_amd_avx2(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q6_K_amd_avx(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q6_K_arm80(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_q8_K_amd_avx512(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_K_amd_avx2(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_K_amd_avx(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_K_arm80(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq2_xxs_amd_avx512(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xxs_amd_avx2(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xxs_amd_avx(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xxs_arm80(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq2_xs_amd_avx512 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xs_amd_avx2 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xs_amd_avx (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xs_arm80 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq2_s_amd_avx512 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_s_amd_avx2 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_s_amd_avx (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_s_arm80 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq3_xxs_amd_avx512(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_xxs_amd_avx2(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_xxs_amd_avx(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_xxs_arm80(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq1_s_amd_avx512 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_s_amd_avx2 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_s_amd_avx (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_s_arm80 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq1_m_amd_avx512 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_m_amd_avx2 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_m_amd_avx (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_m_arm80 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq4_nl_amd_avx512 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_nl_amd_avx2 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_nl_amd_avx (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_nl_arm80 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq4_xs_amd_avx512 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_xs_amd_avx2 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_xs_amd_avx (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_xs_arm80 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void dequantize_row_iq3_s_amd_avx512 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_s_amd_avx2 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_s_amd_avx (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_s_arm80 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_1_q8_1_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_1_q8_1_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q8_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q2_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q3_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q6_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xs_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_m_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_nl_q8_0_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_xs_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+extern "C" size_t quantize_iq2_xxs_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_xxs_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_xxs_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_xxs_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq2_xs_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_xs_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_xs_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_xs_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq2_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq3_xxs_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_xxs_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_xxs_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_xxs_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq1_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq1_m_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_m_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_m_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_m_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq4_nl_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_nl_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_nl_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_nl_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq4_xs_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_xs_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_xs_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_xs_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_iq3_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q2_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q2_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q2_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q2_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q3_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q3_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q3_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q3_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q4_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q5_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q6_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q6_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q6_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q6_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q4_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q4_1_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_1_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_1_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_1_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q5_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q5_1_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_1_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_1_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_1_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" size_t quantize_q8_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q8_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q8_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q8_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+extern "C" void iq2xs_init_impl_amd_avx512(enum ggml_type type);
+extern "C" void iq2xs_init_impl_amd_avx2(enum ggml_type type);
+extern "C" void iq2xs_init_impl_amd_avx(enum ggml_type type);
+extern "C" void iq2xs_init_impl_arm80(enum ggml_type type);
+
+extern "C" void iq2xs_free_impl_amd_avx512(enum ggml_type type);
+extern "C" void iq2xs_free_impl_amd_avx2(enum ggml_type type);
+extern "C" void iq2xs_free_impl_amd_avx(enum ggml_type type);
+extern "C" void iq2xs_free_impl_arm80(enum ggml_type type);
+
+extern "C" void iq3xs_init_impl_amd_avx512(int grid_size);
+extern "C" void iq3xs_init_impl_amd_avx2(int grid_size);
+extern "C" void iq3xs_init_impl_amd_avx(int grid_size);
+extern "C" void iq3xs_init_impl_arm80(int grid_size);
+
+extern "C" void iq3xs_free_impl_amd_avx512(int grid_size);
+extern "C" void iq3xs_free_impl_amd_avx2(int grid_size);
+extern "C" void iq3xs_free_impl_amd_avx(int grid_size);
+extern "C" void iq3xs_free_impl_arm80(int grid_size);
+
+extern "C" bool ggml_validate_row_data_amd_avx512(enum ggml_type type, const void * data, size_t nbytes);
+extern "C" bool ggml_validate_row_data_amd_avx2(enum ggml_type type, const void * data, size_t nbytes);
+extern "C" bool ggml_validate_row_data_amd_avx(enum ggml_type type, const void * data, size_t nbytes);
+extern "C" bool ggml_validate_row_data_arm80(enum ggml_type type, const void * data, size_t nbytes);
+
+static const struct QuantFuncs {
+ typeof(quantize_row_q4_0_reference) *ptr_quantize_row_q4_0_reference;
+ typeof(quantize_row_q4_1_reference) *ptr_quantize_row_q4_1_reference;
+ typeof(quantize_row_q5_0_reference) *ptr_quantize_row_q5_0_reference;
+ typeof(quantize_row_q5_1_reference) *ptr_quantize_row_q5_1_reference;
+ typeof(quantize_row_q8_0_reference) *ptr_quantize_row_q8_0_reference;
+ typeof(quantize_row_q8_1_reference) *ptr_quantize_row_q8_1_reference;
+ typeof(quantize_row_q2_K_reference) *ptr_quantize_row_q2_K_reference;
+ typeof(quantize_row_q3_K_reference) *ptr_quantize_row_q3_K_reference;
+ typeof(quantize_row_q4_K_reference) *ptr_quantize_row_q4_K_reference;
+ typeof(quantize_row_q5_K_reference) *ptr_quantize_row_q5_K_reference;
+ typeof(quantize_row_q6_K_reference) *ptr_quantize_row_q6_K_reference;
+ typeof(quantize_row_q8_K_reference) *ptr_quantize_row_q8_K_reference;
+ typeof(quantize_row_iq3_xxs_reference) *ptr_quantize_row_iq3_xxs_reference;
+ typeof(quantize_row_iq4_nl_reference) *ptr_quantize_row_iq4_nl_reference;
+ typeof(quantize_row_iq4_xs_reference) *ptr_quantize_row_iq4_xs_reference;
+ typeof(quantize_row_iq3_s_reference) *ptr_quantize_row_iq3_s_reference;
+ typeof(quantize_row_iq2_s_reference) *ptr_quantize_row_iq2_s_reference;
+ typeof(quantize_row_q4_0) *ptr_quantize_row_q4_0;
+ typeof(quantize_row_q4_1) *ptr_quantize_row_q4_1;
+ typeof(quantize_row_q5_0) *ptr_quantize_row_q5_0;
+ typeof(quantize_row_q5_1) *ptr_quantize_row_q5_1;
+ typeof(quantize_row_q8_0) *ptr_quantize_row_q8_0;
+ typeof(quantize_row_q8_1) *ptr_quantize_row_q8_1;
+ typeof(quantize_row_q2_K) *ptr_quantize_row_q2_K;
+ typeof(quantize_row_q3_K) *ptr_quantize_row_q3_K;
+ typeof(quantize_row_q4_K) *ptr_quantize_row_q4_K;
+ typeof(quantize_row_q5_K) *ptr_quantize_row_q5_K;
+ typeof(quantize_row_q6_K) *ptr_quantize_row_q6_K;
+ typeof(quantize_row_q8_K) *ptr_quantize_row_q8_K;
+ typeof(quantize_row_iq3_xxs) *ptr_quantize_row_iq3_xxs;
+ typeof(quantize_row_iq4_nl) *ptr_quantize_row_iq4_nl;
+ typeof(quantize_row_iq4_xs) *ptr_quantize_row_iq4_xs;
+ typeof(quantize_row_iq3_s) *ptr_quantize_row_iq3_s;
+ typeof(quantize_row_iq2_s) *ptr_quantize_row_iq2_s;
+ typeof(dequantize_row_q4_0) *ptr_dequantize_row_q4_0;
+ typeof(dequantize_row_q4_1) *ptr_dequantize_row_q4_1;
+ typeof(dequantize_row_q5_0) *ptr_dequantize_row_q5_0;
+ typeof(dequantize_row_q5_1) *ptr_dequantize_row_q5_1;
+ typeof(dequantize_row_q8_0) *ptr_dequantize_row_q8_0;
+ typeof(dequantize_row_q2_K) *ptr_dequantize_row_q2_K;
+ typeof(dequantize_row_q3_K) *ptr_dequantize_row_q3_K;
+ typeof(dequantize_row_q4_K) *ptr_dequantize_row_q4_K;
+ typeof(dequantize_row_q5_K) *ptr_dequantize_row_q5_K;
+ typeof(dequantize_row_q6_K) *ptr_dequantize_row_q6_K;
+ typeof(dequantize_row_q8_K) *ptr_dequantize_row_q8_K;
+ typeof(dequantize_row_iq2_xxs) *ptr_dequantize_row_iq2_xxs;
+ typeof(dequantize_row_iq2_xs) *ptr_dequantize_row_iq2_xs;
+ typeof(dequantize_row_iq2_s) *ptr_dequantize_row_iq2_s;
+ typeof(dequantize_row_iq3_xxs) *ptr_dequantize_row_iq3_xxs;
+ typeof(dequantize_row_iq1_s) *ptr_dequantize_row_iq1_s;
+ typeof(dequantize_row_iq1_m) *ptr_dequantize_row_iq1_m;
+ typeof(dequantize_row_iq4_nl) *ptr_dequantize_row_iq4_nl;
+ typeof(dequantize_row_iq4_xs) *ptr_dequantize_row_iq4_xs;
+ typeof(dequantize_row_iq3_s) *ptr_dequantize_row_iq3_s;
+ typeof(ggml_vec_dot_q4_0_q8_0) *ptr_ggml_vec_dot_q4_0_q8_0;
+ typeof(ggml_vec_dot_q4_1_q8_1) *ptr_ggml_vec_dot_q4_1_q8_1;
+ typeof(ggml_vec_dot_q5_0_q8_0) *ptr_ggml_vec_dot_q5_0_q8_0;
+ typeof(ggml_vec_dot_q5_1_q8_1) *ptr_ggml_vec_dot_q5_1_q8_1;
+ typeof(ggml_vec_dot_q8_0_q8_0) *ptr_ggml_vec_dot_q8_0_q8_0;
+ typeof(ggml_vec_dot_q2_K_q8_K) *ptr_ggml_vec_dot_q2_K_q8_K;
+ typeof(ggml_vec_dot_q3_K_q8_K) *ptr_ggml_vec_dot_q3_K_q8_K;
+ typeof(ggml_vec_dot_q4_K_q8_K) *ptr_ggml_vec_dot_q4_K_q8_K;
+ typeof(ggml_vec_dot_q5_K_q8_K) *ptr_ggml_vec_dot_q5_K_q8_K;
+ typeof(ggml_vec_dot_q6_K_q8_K) *ptr_ggml_vec_dot_q6_K_q8_K;
+ typeof(ggml_vec_dot_iq2_xxs_q8_K) *ptr_ggml_vec_dot_iq2_xxs_q8_K;
+ typeof(ggml_vec_dot_iq2_xs_q8_K) *ptr_ggml_vec_dot_iq2_xs_q8_K;
+ typeof(ggml_vec_dot_iq2_s_q8_K) *ptr_ggml_vec_dot_iq2_s_q8_K;
+ typeof(ggml_vec_dot_iq3_xxs_q8_K) *ptr_ggml_vec_dot_iq3_xxs_q8_K;
+ typeof(ggml_vec_dot_iq1_s_q8_K) *ptr_ggml_vec_dot_iq1_s_q8_K;
+ typeof(ggml_vec_dot_iq1_m_q8_K) *ptr_ggml_vec_dot_iq1_m_q8_K;
+ typeof(ggml_vec_dot_iq4_nl_q8_0) *ptr_ggml_vec_dot_iq4_nl_q8_0;
+ typeof(ggml_vec_dot_iq4_xs_q8_K) *ptr_ggml_vec_dot_iq4_xs_q8_K;
+ typeof(ggml_vec_dot_iq3_s_q8_K) *ptr_ggml_vec_dot_iq3_s_q8_K;
+ typeof(quantize_iq2_xxs) *ptr_quantize_iq2_xxs;
+ typeof(quantize_iq2_xs) *ptr_quantize_iq2_xs;
+ typeof(quantize_iq2_s) *ptr_quantize_iq2_s;
+ typeof(quantize_iq3_xxs) *ptr_quantize_iq3_xxs;
+ typeof(quantize_iq1_s) *ptr_quantize_iq1_s;
+ typeof(quantize_iq1_m) *ptr_quantize_iq1_m;
+ typeof(quantize_iq4_nl) *ptr_quantize_iq4_nl;
+ typeof(quantize_iq4_xs) *ptr_quantize_iq4_xs;
+ typeof(quantize_iq3_s) *ptr_quantize_iq3_s;
+ typeof(quantize_q2_K) *ptr_quantize_q2_K;
+ typeof(quantize_q3_K) *ptr_quantize_q3_K;
+ typeof(quantize_q4_K) *ptr_quantize_q4_K;
+ typeof(quantize_q5_K) *ptr_quantize_q5_K;
+ typeof(quantize_q6_K) *ptr_quantize_q6_K;
+ typeof(quantize_q4_0) *ptr_quantize_q4_0;
+ typeof(quantize_q4_1) *ptr_quantize_q4_1;
+ typeof(quantize_q5_0) *ptr_quantize_q5_0;
+ typeof(quantize_q5_1) *ptr_quantize_q5_1;
+ typeof(quantize_q8_0) *ptr_quantize_q8_0;
+ typeof(iq2xs_init_impl) *ptr_iq2xs_init_impl;
+ typeof(iq2xs_free_impl) *ptr_iq2xs_free_impl;
+ typeof(iq3xs_init_impl) *ptr_iq3xs_init_impl;
+ typeof(iq3xs_free_impl) *ptr_iq3xs_free_impl;
+ typeof(ggml_validate_row_data) *ptr_ggml_validate_row_data;
+
+ QuantFuncs() {
+#ifdef __x86_64__
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F)) {
+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_amd_avx512;
+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_amd_avx512;
+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_amd_avx512;
+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_amd_avx512;
+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_amd_avx512;
+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_amd_avx512;
+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_amd_avx512;
+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_amd_avx512;
+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_amd_avx512;
+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_amd_avx512;
+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_amd_avx512;
+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_amd_avx512;
+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_amd_avx512;
+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_amd_avx512;
+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_amd_avx512;
+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_amd_avx512;
+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_amd_avx512;
+ ptr_quantize_row_q4_0 = quantize_row_q4_0_amd_avx512;
+ ptr_quantize_row_q4_1 = quantize_row_q4_1_amd_avx512;
+ ptr_quantize_row_q5_0 = quantize_row_q5_0_amd_avx512;
+ ptr_quantize_row_q5_1 = quantize_row_q5_1_amd_avx512;
+ ptr_quantize_row_q8_0 = quantize_row_q8_0_amd_avx512;
+ ptr_quantize_row_q8_1 = quantize_row_q8_1_amd_avx512;
+ ptr_quantize_row_q2_K = quantize_row_q2_K_amd_avx512;
+ ptr_quantize_row_q3_K = quantize_row_q3_K_amd_avx512;
+ ptr_quantize_row_q4_K = quantize_row_q4_K_amd_avx512;
+ ptr_quantize_row_q5_K = quantize_row_q5_K_amd_avx512;
+ ptr_quantize_row_q6_K = quantize_row_q6_K_amd_avx512;
+ ptr_quantize_row_q8_K = quantize_row_q8_K_amd_avx512;
+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_amd_avx512;
+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_amd_avx512;
+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_amd_avx512;
+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_amd_avx512;
+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_amd_avx512;
+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_amd_avx512;
+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_amd_avx512;
+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_amd_avx512;
+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_amd_avx512;
+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_amd_avx512;
+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_amd_avx512;
+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_amd_avx512;
+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_amd_avx512;
+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_amd_avx512;
+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_amd_avx512;
+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_amd_avx512;
+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_amd_avx512;
+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_amd_avx512;
+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_amd_avx512;
+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_amd_avx512;
+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_amd_avx512;
+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_amd_avx512;
+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_amd_avx512;
+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_amd_avx512;
+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_amd_avx512;
+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_amd_avx512;
+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_amd_avx512;
+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_amd_avx512;
+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_amd_avx512;
+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_amd_avx512;
+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_amd_avx512;
+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_amd_avx512;
+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_amd_avx512;
+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_amd_avx512;
+ ptr_quantize_iq2_xs = quantize_iq2_xs_amd_avx512;
+ ptr_quantize_iq2_s = quantize_iq2_s_amd_avx512;
+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_amd_avx512;
+ ptr_quantize_iq1_s = quantize_iq1_s_amd_avx512;
+ ptr_quantize_iq1_m = quantize_iq1_m_amd_avx512;
+ ptr_quantize_iq4_nl = quantize_iq4_nl_amd_avx512;
+ ptr_quantize_iq4_xs = quantize_iq4_xs_amd_avx512;
+ ptr_quantize_iq3_s = quantize_iq3_s_amd_avx512;
+ ptr_quantize_q2_K = quantize_q2_K_amd_avx512;
+ ptr_quantize_q3_K = quantize_q3_K_amd_avx512;
+ ptr_quantize_q4_K = quantize_q4_K_amd_avx512;
+ ptr_quantize_q5_K = quantize_q5_K_amd_avx512;
+ ptr_quantize_q6_K = quantize_q6_K_amd_avx512;
+ ptr_quantize_q4_0 = quantize_q4_0_amd_avx512;
+ ptr_quantize_q4_1 = quantize_q4_1_amd_avx512;
+ ptr_quantize_q5_0 = quantize_q5_0_amd_avx512;
+ ptr_quantize_q5_1 = quantize_q5_1_amd_avx512;
+ ptr_quantize_q8_0 = quantize_q8_0_amd_avx512;
+ ptr_iq2xs_init_impl = iq2xs_init_impl_amd_avx512;
+ ptr_iq2xs_free_impl = iq2xs_free_impl_amd_avx512;
+ ptr_iq3xs_init_impl = iq3xs_init_impl_amd_avx512;
+ ptr_iq3xs_free_impl = iq3xs_free_impl_amd_avx512;
+ ptr_ggml_validate_row_data = ggml_validate_row_data_amd_avx512;
+ return;
+ }
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2)) {
+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_amd_avx2;
+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_amd_avx2;
+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_amd_avx2;
+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_amd_avx2;
+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_amd_avx2;
+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_amd_avx2;
+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_amd_avx2;
+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_amd_avx2;
+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_amd_avx2;
+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_amd_avx2;
+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_amd_avx2;
+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_amd_avx2;
+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_amd_avx2;
+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_amd_avx2;
+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_amd_avx2;
+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_amd_avx2;
+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_amd_avx2;
+ ptr_quantize_row_q4_0 = quantize_row_q4_0_amd_avx2;
+ ptr_quantize_row_q4_1 = quantize_row_q4_1_amd_avx2;
+ ptr_quantize_row_q5_0 = quantize_row_q5_0_amd_avx2;
+ ptr_quantize_row_q5_1 = quantize_row_q5_1_amd_avx2;
+ ptr_quantize_row_q8_0 = quantize_row_q8_0_amd_avx2;
+ ptr_quantize_row_q8_1 = quantize_row_q8_1_amd_avx2;
+ ptr_quantize_row_q2_K = quantize_row_q2_K_amd_avx2;
+ ptr_quantize_row_q3_K = quantize_row_q3_K_amd_avx2;
+ ptr_quantize_row_q4_K = quantize_row_q4_K_amd_avx2;
+ ptr_quantize_row_q5_K = quantize_row_q5_K_amd_avx2;
+ ptr_quantize_row_q6_K = quantize_row_q6_K_amd_avx2;
+ ptr_quantize_row_q8_K = quantize_row_q8_K_amd_avx2;
+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_amd_avx2;
+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_amd_avx2;
+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_amd_avx2;
+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_amd_avx2;
+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_amd_avx2;
+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_amd_avx2;
+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_amd_avx2;
+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_amd_avx2;
+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_amd_avx2;
+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_amd_avx2;
+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_amd_avx2;
+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_amd_avx2;
+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_amd_avx2;
+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_amd_avx2;
+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_amd_avx2;
+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_amd_avx2;
+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_amd_avx2;
+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_amd_avx2;
+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_amd_avx2;
+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_amd_avx2;
+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_amd_avx2;
+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_amd_avx2;
+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_amd_avx2;
+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_amd_avx2;
+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_amd_avx2;
+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_amd_avx2;
+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_amd_avx2;
+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_amd_avx2;
+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_amd_avx2;
+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_amd_avx2;
+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_amd_avx2;
+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_amd_avx2;
+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_amd_avx2;
+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_amd_avx2;
+ ptr_quantize_iq2_xs = quantize_iq2_xs_amd_avx2;
+ ptr_quantize_iq2_s = quantize_iq2_s_amd_avx2;
+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_amd_avx2;
+ ptr_quantize_iq1_s = quantize_iq1_s_amd_avx2;
+ ptr_quantize_iq1_m = quantize_iq1_m_amd_avx2;
+ ptr_quantize_iq4_nl = quantize_iq4_nl_amd_avx2;
+ ptr_quantize_iq4_xs = quantize_iq4_xs_amd_avx2;
+ ptr_quantize_iq3_s = quantize_iq3_s_amd_avx2;
+ ptr_quantize_q2_K = quantize_q2_K_amd_avx2;
+ ptr_quantize_q3_K = quantize_q3_K_amd_avx2;
+ ptr_quantize_q4_K = quantize_q4_K_amd_avx2;
+ ptr_quantize_q5_K = quantize_q5_K_amd_avx2;
+ ptr_quantize_q6_K = quantize_q6_K_amd_avx2;
+ ptr_quantize_q4_0 = quantize_q4_0_amd_avx2;
+ ptr_quantize_q4_1 = quantize_q4_1_amd_avx2;
+ ptr_quantize_q5_0 = quantize_q5_0_amd_avx2;
+ ptr_quantize_q5_1 = quantize_q5_1_amd_avx2;
+ ptr_quantize_q8_0 = quantize_q8_0_amd_avx2;
+ ptr_iq2xs_init_impl = iq2xs_init_impl_amd_avx2;
+ ptr_iq2xs_free_impl = iq2xs_free_impl_amd_avx2;
+ ptr_iq3xs_init_impl = iq3xs_init_impl_amd_avx2;
+ ptr_iq3xs_free_impl = iq3xs_free_impl_amd_avx2;
+ ptr_ggml_validate_row_data = ggml_validate_row_data_amd_avx2;
+ return;
+ }
+ if (1) {
+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_amd_avx;
+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_amd_avx;
+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_amd_avx;
+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_amd_avx;
+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_amd_avx;
+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_amd_avx;
+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_amd_avx;
+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_amd_avx;
+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_amd_avx;
+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_amd_avx;
+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_amd_avx;
+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_amd_avx;
+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_amd_avx;
+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_amd_avx;
+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_amd_avx;
+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_amd_avx;
+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_amd_avx;
+ ptr_quantize_row_q4_0 = quantize_row_q4_0_amd_avx;
+ ptr_quantize_row_q4_1 = quantize_row_q4_1_amd_avx;
+ ptr_quantize_row_q5_0 = quantize_row_q5_0_amd_avx;
+ ptr_quantize_row_q5_1 = quantize_row_q5_1_amd_avx;
+ ptr_quantize_row_q8_0 = quantize_row_q8_0_amd_avx;
+ ptr_quantize_row_q8_1 = quantize_row_q8_1_amd_avx;
+ ptr_quantize_row_q2_K = quantize_row_q2_K_amd_avx;
+ ptr_quantize_row_q3_K = quantize_row_q3_K_amd_avx;
+ ptr_quantize_row_q4_K = quantize_row_q4_K_amd_avx;
+ ptr_quantize_row_q5_K = quantize_row_q5_K_amd_avx;
+ ptr_quantize_row_q6_K = quantize_row_q6_K_amd_avx;
+ ptr_quantize_row_q8_K = quantize_row_q8_K_amd_avx;
+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_amd_avx;
+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_amd_avx;
+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_amd_avx;
+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_amd_avx;
+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_amd_avx;
+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_amd_avx;
+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_amd_avx;
+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_amd_avx;
+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_amd_avx;
+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_amd_avx;
+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_amd_avx;
+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_amd_avx;
+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_amd_avx;
+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_amd_avx;
+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_amd_avx;
+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_amd_avx;
+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_amd_avx;
+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_amd_avx;
+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_amd_avx;
+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_amd_avx;
+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_amd_avx;
+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_amd_avx;
+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_amd_avx;
+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_amd_avx;
+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_amd_avx;
+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_amd_avx;
+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_amd_avx;
+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_amd_avx;
+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_amd_avx;
+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_amd_avx;
+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_amd_avx;
+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_amd_avx;
+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_amd_avx;
+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_amd_avx;
+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_amd_avx;
+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_amd_avx;
+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_amd_avx;
+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_amd_avx;
+ ptr_quantize_iq2_xs = quantize_iq2_xs_amd_avx;
+ ptr_quantize_iq2_s = quantize_iq2_s_amd_avx;
+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_amd_avx;
+ ptr_quantize_iq1_s = quantize_iq1_s_amd_avx;
+ ptr_quantize_iq1_m = quantize_iq1_m_amd_avx;
+ ptr_quantize_iq4_nl = quantize_iq4_nl_amd_avx;
+ ptr_quantize_iq4_xs = quantize_iq4_xs_amd_avx;
+ ptr_quantize_iq3_s = quantize_iq3_s_amd_avx;
+ ptr_quantize_q2_K = quantize_q2_K_amd_avx;
+ ptr_quantize_q3_K = quantize_q3_K_amd_avx;
+ ptr_quantize_q4_K = quantize_q4_K_amd_avx;
+ ptr_quantize_q5_K = quantize_q5_K_amd_avx;
+ ptr_quantize_q6_K = quantize_q6_K_amd_avx;
+ ptr_quantize_q4_0 = quantize_q4_0_amd_avx;
+ ptr_quantize_q4_1 = quantize_q4_1_amd_avx;
+ ptr_quantize_q5_0 = quantize_q5_0_amd_avx;
+ ptr_quantize_q5_1 = quantize_q5_1_amd_avx;
+ ptr_quantize_q8_0 = quantize_q8_0_amd_avx;
+ ptr_iq2xs_init_impl = iq2xs_init_impl_amd_avx;
+ ptr_iq2xs_free_impl = iq2xs_free_impl_amd_avx;
+ ptr_iq3xs_init_impl = iq3xs_init_impl_amd_avx;
+ ptr_iq3xs_free_impl = iq3xs_free_impl_amd_avx;
+ ptr_ggml_validate_row_data = ggml_validate_row_data_amd_avx;
+ return;
+ }
+#else
+ ptr_quantize_row_q4_0_reference = quantize_row_q4_0_reference_arm80;
+ ptr_quantize_row_q4_1_reference = quantize_row_q4_1_reference_arm80;
+ ptr_quantize_row_q5_0_reference = quantize_row_q5_0_reference_arm80;
+ ptr_quantize_row_q5_1_reference = quantize_row_q5_1_reference_arm80;
+ ptr_quantize_row_q8_0_reference = quantize_row_q8_0_reference_arm80;
+ ptr_quantize_row_q8_1_reference = quantize_row_q8_1_reference_arm80;
+ ptr_quantize_row_q2_K_reference = quantize_row_q2_K_reference_arm80;
+ ptr_quantize_row_q3_K_reference = quantize_row_q3_K_reference_arm80;
+ ptr_quantize_row_q4_K_reference = quantize_row_q4_K_reference_arm80;
+ ptr_quantize_row_q5_K_reference = quantize_row_q5_K_reference_arm80;
+ ptr_quantize_row_q6_K_reference = quantize_row_q6_K_reference_arm80;
+ ptr_quantize_row_q8_K_reference = quantize_row_q8_K_reference_arm80;
+ ptr_quantize_row_iq3_xxs_reference = quantize_row_iq3_xxs_reference_arm80;
+ ptr_quantize_row_iq4_nl_reference = quantize_row_iq4_nl_reference_arm80;
+ ptr_quantize_row_iq4_xs_reference = quantize_row_iq4_xs_reference_arm80;
+ ptr_quantize_row_iq3_s_reference = quantize_row_iq3_s_reference_arm80;
+ ptr_quantize_row_iq2_s_reference = quantize_row_iq2_s_reference_arm80;
+ ptr_quantize_row_q4_0 = quantize_row_q4_0_arm80;
+ ptr_quantize_row_q4_1 = quantize_row_q4_1_arm80;
+ ptr_quantize_row_q5_0 = quantize_row_q5_0_arm80;
+ ptr_quantize_row_q5_1 = quantize_row_q5_1_arm80;
+ ptr_quantize_row_q8_0 = quantize_row_q8_0_arm80;
+ ptr_quantize_row_q8_1 = quantize_row_q8_1_arm80;
+ ptr_quantize_row_q2_K = quantize_row_q2_K_arm80;
+ ptr_quantize_row_q3_K = quantize_row_q3_K_arm80;
+ ptr_quantize_row_q4_K = quantize_row_q4_K_arm80;
+ ptr_quantize_row_q5_K = quantize_row_q5_K_arm80;
+ ptr_quantize_row_q6_K = quantize_row_q6_K_arm80;
+ ptr_quantize_row_q8_K = quantize_row_q8_K_arm80;
+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_arm80;
+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_arm80;
+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_arm80;
+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_arm80;
+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_arm80;
+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_arm80;
+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_arm80;
+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_arm80;
+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_arm80;
+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_arm80;
+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_arm80;
+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_arm80;
+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_arm80;
+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_arm80;
+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_arm80;
+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_arm80;
+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_arm80;
+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_arm80;
+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_arm80;
+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_arm80;
+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_arm80;
+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_arm80;
+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_arm80;
+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_arm80;
+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_arm80;
+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_arm80;
+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_arm80;
+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_arm80;
+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_arm80;
+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_arm80;
+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_arm80;
+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_arm80;
+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_arm80;
+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_arm80;
+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_arm80;
+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_arm80;
+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_arm80;
+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_arm80;
+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_arm80;
+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_arm80;
+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_arm80;
+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_arm80;
+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_arm80;
+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_arm80;
+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_arm80;
+ ptr_quantize_iq2_xs = quantize_iq2_xs_arm80;
+ ptr_quantize_iq2_s = quantize_iq2_s_arm80;
+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_arm80;
+ ptr_quantize_iq1_s = quantize_iq1_s_arm80;
+ ptr_quantize_iq1_m = quantize_iq1_m_arm80;
+ ptr_quantize_iq4_nl = quantize_iq4_nl_arm80;
+ ptr_quantize_iq4_xs = quantize_iq4_xs_arm80;
+ ptr_quantize_iq3_s = quantize_iq3_s_arm80;
+ ptr_quantize_q2_K = quantize_q2_K_arm80;
+ ptr_quantize_q3_K = quantize_q3_K_arm80;
+ ptr_quantize_q4_K = quantize_q4_K_arm80;
+ ptr_quantize_q5_K = quantize_q5_K_arm80;
+ ptr_quantize_q6_K = quantize_q6_K_arm80;
+ ptr_quantize_q4_0 = quantize_q4_0_arm80;
+ ptr_quantize_q4_1 = quantize_q4_1_arm80;
+ ptr_quantize_q5_0 = quantize_q5_0_arm80;
+ ptr_quantize_q5_1 = quantize_q5_1_arm80;
+ ptr_quantize_q8_0 = quantize_q8_0_arm80;
+ ptr_iq2xs_init_impl = iq2xs_init_impl_arm80;
+ ptr_iq2xs_free_impl = iq2xs_free_impl_arm80;
+ ptr_iq3xs_init_impl = iq3xs_init_impl_arm80;
+ ptr_iq3xs_free_impl = iq3xs_free_impl_arm80;
+ ptr_ggml_validate_row_data = ggml_validate_row_data_arm80;
+#endif
+ }
+} funcs;
+
+void quantize_row_q4_0_reference(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q4_0_reference(x, y, k);
+}
+
+void quantize_row_q4_1_reference(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q4_1_reference(x, y, k);
+}
+
+void quantize_row_q5_0_reference(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q5_0_reference(x, y, k);
+}
+
+void quantize_row_q5_1_reference(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q5_1_reference(x, y, k);
+}
+
+void quantize_row_q8_0_reference(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q8_0_reference(x, y, k);
+}
+
+void quantize_row_q8_1_reference(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q8_1_reference(x, y, k);
+}
+
+void quantize_row_q2_K_reference(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q2_K_reference(x, y, k);
+}
+
+void quantize_row_q3_K_reference(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q3_K_reference(x, y, k);
+}
+
+void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q4_K_reference(x, y, k);
+}
+
+void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q5_K_reference(x, y, k);
+}
+
+void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q6_K_reference(x, y, k);
+}
+
+void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q8_K_reference(x, y, k);
+}
+
+void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq3_xxs_reference(x, y, k);
+}
+
+void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq4_nl_reference(x, y, k);
+}
+
+void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq4_xs_reference(x, y, k);
+}
+
+void quantize_row_iq3_s_reference (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq3_s_reference(x, y, k);
+}
+
+void quantize_row_iq2_s_reference (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq2_s_reference(x, y, k);
+}
+
+void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q4_0(x, y, k);
+}
+
+void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q4_1(x, y, k);
+}
+
+void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q5_0(x, y, k);
+}
+
+void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q5_1(x, y, k);
+}
+
+void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q8_0(x, y, k);
+}
+
+void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q8_1(x, y, k);
+}
+
+void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q2_K(x, y, k);
+}
+
+void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q3_K(x, y, k);
+}
+
+void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q4_K(x, y, k);
+}
+
+void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q5_K(x, y, k);
+}
+
+void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q6_K(x, y, k);
+}
+
+void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_q8_K(x, y, k);
+}
+
+void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq3_xxs(x, y, k);
+}
+
+void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq4_nl(x, y, k);
+}
+
+void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq4_xs(x, y, k);
+}
+
+void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq3_s(x, y, k);
+}
+
+void quantize_row_iq2_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_quantize_row_iq2_s(x, y, k);
+}
+
+void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q4_0(x, y, k);
+}
+
+void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q4_1(x, y, k);
+}
+
+void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q5_0(x, y, k);
+}
+
+void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q5_1(x, y, k);
+}
+
+void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q8_0(x, y, k);
+}
+
+void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q2_K(x, y, k);
+}
+
+void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q3_K(x, y, k);
+}
+
+void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q4_K(x, y, k);
+}
+
+void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q5_K(x, y, k);
+}
+
+void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q6_K(x, y, k);
+}
+
+void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_q8_K(x, y, k);
+}
+
+void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq2_xxs(x, y, k);
+}
+
+void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq2_xs(x, y, k);
+}
+
+void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq2_s(x, y, k);
+}
+
+void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq3_xxs(x, y, k);
+}
+
+void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq1_s(x, y, k);
+}
+
+void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq1_m(x, y, k);
+}
+
+void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq4_nl(x, y, k);
+}
+
+void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq4_xs(x, y, k);
+}
+
+void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ return funcs.ptr_dequantize_row_iq3_s(x, y, k);
+}
+
+void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q4_0_q8_0(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q4_1_q8_1(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q5_0_q8_0(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q5_1_q8_1(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q8_0_q8_0(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q2_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q3_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q4_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q5_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_q6_K_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq2_xxs_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq2_xs_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq2_s_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq3_xxs_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq1_s_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq1_m_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq4_nl_q8_0(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq4_xs_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_iq3_s_q8_K(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq2_xxs(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq2_xs(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq2_s(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq3_xxs(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq1_s(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq1_m(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq4_nl(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq4_xs(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_iq3_s(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q2_K(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q3_K(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q4_K(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q5_K(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q6_K(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q4_0(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q4_1(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q5_0(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q5_1(src, dst, nrows, n_per_row, imatrix);
+}
+
+size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ return funcs.ptr_quantize_q8_0(src, dst, nrows, n_per_row, imatrix);
+}
+
+void iq2xs_init_impl(enum ggml_type type) {
+ return funcs.ptr_iq2xs_init_impl(type);
+}
+
+void iq2xs_free_impl(enum ggml_type type) {
+ return funcs.ptr_iq2xs_free_impl(type);
+}
+
+void iq3xs_init_impl(int grid_size) {
+ return funcs.ptr_iq3xs_init_impl(grid_size);
+}
+
+void iq3xs_free_impl(int grid_size) {
+ return funcs.ptr_iq3xs_free_impl(grid_size);
+}
+
+bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) {
+ return funcs.ptr_ggml_validate_row_data(type, data, nbytes);
+}
+
diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.inc
similarity index 75%
rename from llama.cpp/ggml-quants.c
rename to llama.cpp/ggml-quants.inc
index f753167c..5850ff47 100644
--- a/llama.cpp/ggml-quants.c
+++ b/llama.cpp/ggml-quants.inc
@@ -1,6 +1,3 @@
-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
-
#define GGML_COMMON_IMPL_C
#include "ggml-common.h"
@@ -16,98 +13,55 @@
#include <float.h>
#include <stdlib.h> // for qsort
#include <stdio.h> // for GGML_ASSERT
-#include <cosmo.h>
-
-#ifdef __ARM_NEON
-
-// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
-//
-// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
-//
-#include <arm_neon.h>
-
-#else
-
-#ifdef __wasm_simd128__
-#include <wasm_simd128.h>
-#else
-#if defined(__POWER9_VECTOR__) || defined(__powerpc64__)
-#include <altivec.h>
-#undef bool
-#define bool _Bool
-#else
-#if defined(_MSC_VER) || defined(__MINGW32__)
-#include <intrin.h>
-#else
-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) || defined(__SSE3__)
-#if !defined(__riscv)
-#include <immintrin.h>
-#endif
-#endif
-#endif
-#endif
-#endif
-#endif
-
-#ifdef __riscv_v_intrinsic
-#include <riscv_vector.h>
-#endif
-
-#undef MIN
-#undef MAX
-
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define UNUSED GGML_UNUSED
// some compilers don't provide _mm256_set_m128i, e.g. gcc 7
#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)
+// multiply int8_t, add results pairwise twice
+static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {
+ // Get absolute values of x vectors
+ const __m128i ax = _mm_sign_epi8(x, x);
+ // Sign the values of the y vectors
+ const __m128i sy = _mm_sign_epi8(y, x);
+ // Perform multiplication and create 16-bit values
+ const __m128i dot = _mm_maddubs_epi16(ax, sy);
+ const __m128i ones = _mm_set1_epi16(1);
+ return _mm_madd_epi16(ones, dot);
+}
+
+#if __AVX__ || __AVX2__ || __AVX512F__
// horizontally add 8 floats
-#define hsum_float_8(x) ({ \
- __m128 res = _mm256_extractf128_ps(x, 1); \
- res = _mm_add_ps(res, _mm256_castps256_ps128(x)); \
- res = _mm_add_ps(res, _mm_movehl_ps(res, res)); \
- res = _mm_add_ss(res, _mm_movehdup_ps(res)); \
- _mm_cvtss_f32(res); \
-})
-
-#ifdef __x86_64__
-
-#pragma GCC push_options
-#pragma GCC target("avx2")
-// Unpack 32 4-bit fields into 32 bytes
-// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
-static inline __m256i bytes_from_nibbles_32_avx2(const uint8_t * rsi)
-{
- const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);
- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
- const __m256i lowMask = _mm256_set1_epi8( 0xF );
- return _mm256_and_si256(lowMask, bytes);
+static inline float hsum_float_8(const __m256 x) {
+ __m128 res = _mm256_extractf128_ps(x, 1);
+ res = _mm_add_ps(res, _mm256_castps256_ps128(x));
+ res = _mm_add_ps(res, _mm_movehl_ps(res, res));
+ res = _mm_add_ss(res, _mm_movehdup_ps(res));
+ return _mm_cvtss_f32(res);
}
-#pragma GCC pop_options
-#pragma GCC push_options
-#pragma GCC target("avx")
-// Unpack 32 4-bit fields into 32 bytes
-// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
-static inline __m256i bytes_from_nibbles_32_avx(const uint8_t * rsi)
-{
- // Load 16 bytes from memory
- __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);
- __m128i tmph = _mm_srli_epi16(tmpl, 4);
- const __m128i lowMask = _mm_set1_epi8(0xF);
- tmpl = _mm_and_si128(lowMask, tmpl);
- tmph = _mm_and_si128(lowMask, tmph);
- return MM256_SET_M128I(tmph, tmpl);
+// horizontally add 8 int32_t
+static inline int hsum_i32_8(const __m256i a) {
+ const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));
+ const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);
+ const __m128i sum64 = _mm_add_epi32(hi64, sum128);
+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
+}
+
+// horizontally add 4 int32_t
+static inline int hsum_i32_4(const __m128i a) {
+ const __m128i hi64 = _mm_unpackhi_epi64(a, a);
+ const __m128i sum64 = _mm_add_epi32(hi64, a);
+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
+ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
}
-#pragma GCC pop_options
-#pragma GCC push_options
-#pragma GCC target("avx2")
+#if defined(__AVX2__) || defined(__AVX512F__)
// spread 32 bits to 32 bytes { 0x00, 0xFF }
-static inline __m256i bytes_from_bits_32_avx2(const uint8_t * x) {
+static inline __m256i bytes_from_bits_32(const uint8_t * x) {
uint32_t x32;
memcpy(&x32, x, sizeof(uint32_t));
const __m256i shuf_mask = _mm256_set_epi64x(
@@ -118,67 +72,38 @@ static inline __m256i bytes_from_bits_32_avx2(const uint8_t * x) {
bytes = _mm256_or_si256(bytes, bit_mask);
return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));
}
-#pragma GCC pop_options
-
-#pragma GCC push_options
-#pragma GCC target("avx")
-// spread 32 bits to 32 bytes { 0x00, 0xFF }
-static inline __m256i bytes_from_bits_32_avx(const uint8_t * x) {
- uint32_t x32;
- memcpy(&x32, x, sizeof(uint32_t));
- const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
- const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);
- __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);
- __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);
- const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);
- bytesl = _mm_or_si128(bytesl, bit_mask);
- bytesh = _mm_or_si128(bytesh, bit_mask);
- bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));
- bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));
- return MM256_SET_M128I(bytesh, bytesl);
-}
-#pragma GCC pop_options
-// add int16_t pairwise and return as float vector
-__attribute__((__target__("avx")))
-static inline __m256 sum_i16_pairs_float_avx(const __m128i xh, const __m128i xl) {
- const __m128i ones = _mm_set1_epi16(1);
- const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);
- const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);
- const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);
- return _mm256_cvtepi32_ps(summed_pairs);
+// Unpack 32 4-bit fields into 32 bytes
+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)
+{
+ const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);
+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
+ const __m256i lowMask = _mm256_set1_epi8( 0xF );
+ return _mm256_and_si256(lowMask, bytes);
}
// add int16_t pairwise and return as float vector
-__attribute__((__target__("avx2,fma")))
-static inline __m256 sum_i16_pairs_float_avx2(const __m256i x) {
+static inline __m256 sum_i16_pairs_float(const __m256i x) {
const __m256i ones = _mm256_set1_epi16(1);
const __m256i summed_pairs = _mm256_madd_epi16(ones, x);
return _mm256_cvtepi32_ps(summed_pairs);
}
-__attribute__((__target__("avx2,fma,f16c")))
-static inline __m256 mul_sum_us8_pairs_float_avx2(const __m256i ax, const __m256i sy) {
+static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {
+#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
+ const __m256i zero = _mm256_setzero_si256();
+ const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);
+ return _mm256_cvtepi32_ps(summed_pairs);
+#else
// Perform multiplication and create 16-bit values
const __m256i dot = _mm256_maddubs_epi16(ax, sy);
- return sum_i16_pairs_float_avx2(dot);
-}
-
-__attribute__((__target__("avx")))
-static inline __m256 mul_sum_us8_pairs_float_avx(const __m256i ax, const __m256i sy) {
- const __m128i axl = _mm256_castsi256_si128(ax);
- const __m128i axh = _mm256_extractf128_si256(ax, 1);
- const __m128i syl = _mm256_castsi256_si128(sy);
- const __m128i syh = _mm256_extractf128_si256(sy, 1);
- // Perform multiplication and create 16-bit values
- const __m128i dotl = _mm_maddubs_epi16(axl, syl);
- const __m128i doth = _mm_maddubs_epi16(axh, syh);
- return sum_i16_pairs_float_avx(doth, dotl);
+ return sum_i16_pairs_float(dot);
+#endif
}
// multiply int8_t, add results pairwise twice and return as float vector
-__attribute__((__target__("avx2,fma")))
-static inline __m256 mul_sum_i8_pairs_float_avx2(const __m256i x, const __m256i y) {
+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {
#if __AVXVNNIINT8__
const __m256i zero = _mm256_setzero_si256();
const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);
@@ -188,70 +113,8 @@ static inline __m256 mul_sum_i8_pairs_float_avx2(const __m256i x, const __m256i
const __m256i ax = _mm256_sign_epi8(x, x);
// Sign the values of the y vectors
const __m256i sy = _mm256_sign_epi8(y, x);
- return mul_sum_us8_pairs_float_avx2(ax, sy);
-#endif
-}
-
-// multiply int8_t, add results pairwise twice and return as float vector
-__attribute__((__target__("avx")))
-static inline __m256 mul_sum_i8_pairs_float_avx(const __m256i x, const __m256i y) {
- const __m128i xl = _mm256_castsi256_si128(x);
- const __m128i xh = _mm256_extractf128_si256(x, 1);
- const __m128i yl = _mm256_castsi256_si128(y);
- const __m128i yh = _mm256_extractf128_si256(y, 1);
- // Get absolute values of x vectors
- const __m128i axl = _mm_sign_epi8(xl, xl);
- const __m128i axh = _mm_sign_epi8(xh, xh);
- // Sign the values of the y vectors
- const __m128i syl = _mm_sign_epi8(yl, xl);
- const __m128i syh = _mm_sign_epi8(yh, xh);
- // Perform multiplication and create 16-bit values
- const __m128i dotl = _mm_maddubs_epi16(axl, syl);
- const __m128i doth = _mm_maddubs_epi16(axh, syh);
- return sum_i16_pairs_float_avx(doth, dotl);
-}
-
+ return mul_sum_us8_pairs_float(ax, sy);
#endif
-
-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)
-// multiply int8_t, add results pairwise twice
-static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {
- // Get absolute values of x vectors
- const __m128i ax = _mm_sign_epi8(x, x);
- // Sign the values of the y vectors
- const __m128i sy = _mm_sign_epi8(y, x);
- // Perform multiplication and create 16-bit values
- const __m128i dot = _mm_maddubs_epi16(ax, sy);
- const __m128i ones = _mm_set1_epi16(1);
- return _mm_madd_epi16(ones, dot);
-}
-
-#if __AVX__ || __AVX2__ || __AVX512F__
-
-// horizontally add 8 int32_t
-static inline int hsum_i32_8(const __m256i a) {
- const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));
- const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);
- const __m128i sum64 = _mm_add_epi32(hi64, sum128);
- const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
- return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
-}
-
-// horizontally add 4 int32_t
-static inline int hsum_i32_4(const __m128i a) {
- const __m128i hi64 = _mm_unpackhi_epi64(a, a);
- const __m128i sum64 = _mm_add_epi32(hi64, a);
- const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
- return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
-}
-
-#if defined(__AVX2__) || defined(__AVX512F__)
-
-// add int16_t pairwise and return as float vector
-static inline __m256 sum_i16_pairs_float(const __m256i x) {
- const __m256i ones = _mm256_set1_epi16(1);
- const __m256i summed_pairs = _mm256_madd_epi16(ones, x);
- return _mm256_cvtepi32_ps(summed_pairs);
}
static inline __m128i packNibbles( __m256i bytes )
@@ -275,6 +138,34 @@ static inline __m128i packNibbles( __m256i bytes )
#endif
}
#elif defined(__AVX__)
+// spread 32 bits to 32 bytes { 0x00, 0xFF }
+static inline __m256i bytes_from_bits_32(const uint8_t * x) {
+ uint32_t x32;
+ memcpy(&x32, x, sizeof(uint32_t));
+ const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
+ const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);
+ __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);
+ __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);
+ const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);
+ bytesl = _mm_or_si128(bytesl, bit_mask);
+ bytesh = _mm_or_si128(bytesh, bit_mask);
+ bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));
+ bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));
+ return MM256_SET_M128I(bytesh, bytesl);
+}
+
+// Unpack 32 4-bit fields into 32 bytes
+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval
+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)
+{
+ // Load 16 bytes from memory
+ __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);
+ __m128i tmph = _mm_srli_epi16(tmpl, 4);
+ const __m128i lowMask = _mm_set1_epi8(0xF);
+ tmpl = _mm_and_si128(lowMask, tmpl);
+ tmph = _mm_and_si128(lowMask, tmph);
+ return MM256_SET_M128I(tmph, tmpl);
+}
// add int16_t pairwise and return as float vector
static inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {
@@ -285,6 +176,35 @@ static inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {
return _mm256_cvtepi32_ps(summed_pairs);
}
+static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {
+ const __m128i axl = _mm256_castsi256_si128(ax);
+ const __m128i axh = _mm256_extractf128_si256(ax, 1);
+ const __m128i syl = _mm256_castsi256_si128(sy);
+ const __m128i syh = _mm256_extractf128_si256(sy, 1);
+ // Perform multiplication and create 16-bit values
+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);
+ const __m128i doth = _mm_maddubs_epi16(axh, syh);
+ return sum_i16_pairs_float(doth, dotl);
+}
+
+// multiply int8_t, add results pairwise twice and return as float vector
+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {
+ const __m128i xl = _mm256_castsi256_si128(x);
+ const __m128i xh = _mm256_extractf128_si256(x, 1);
+ const __m128i yl = _mm256_castsi256_si128(y);
+ const __m128i yh = _mm256_extractf128_si256(y, 1);
+ // Get absolute values of x vectors
+ const __m128i axl = _mm_sign_epi8(xl, xl);
+ const __m128i axh = _mm_sign_epi8(xh, xh);
+ // Sign the values of the y vectors
+ const __m128i syl = _mm_sign_epi8(yl, xl);
+ const __m128i syh = _mm_sign_epi8(yh, xh);
+ // Perform multiplication and create 16-bit values
+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);
+ const __m128i doth = _mm_maddubs_epi16(axh, syh);
+ return sum_i16_pairs_float(doth, dotl);
+}
+
static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )
{
// Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh
@@ -536,67 +456,214 @@ void quantize_row_q8_0_reference(const float * restrict x, block_q8_0 * restrict
}
}
-#ifdef __x86_64__
-#define quantize_row_q8_0 quantize_row_q8_0_default
-#include "quantize_row_q8_0.inc"
-#undef quantize_row_q8_0
-#define quantize_row_q8_0 __attribute__((__target__("avx"))) quantize_row_q8_0_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "quantize_row_q8_0.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef quantize_row_q8_0
-#define quantize_row_q8_0 __attribute__((__target__("avx2,fma,f16c"))) quantize_row_q8_0_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "quantize_row_q8_0.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef quantize_row_q8_0
void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {
- quantize_row_q8_0_avx2(x, vy, k);
- } else if (X86_HAVE(AVX)) {
- quantize_row_q8_0_avx(x, vy, k);
- } else {
- quantize_row_q8_0_default(x, vy, k);
- }
-}
-#else
-#define static
-#include "quantize_row_q8_0.inc"
-#undef static
-#endif
+ assert(QK8_0 == 32);
+ assert(k % QK8_0 == 0);
+ const int nb = k / QK8_0;
-// reference implementation for deterministic creation of model files
-void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict y, int64_t k) {
- assert(QK8_1 == 32);
- assert(k % QK8_1 == 0);
- const int nb = k / QK8_1;
+ block_q8_0 * restrict y = vy;
+#if defined(__ARM_NEON)
for (int i = 0; i < nb; i++) {
- float amax = 0.0f; // absolute max
+ float32x4_t srcv [8];
+ float32x4_t asrcv[8];
+ float32x4_t amaxv[8];
- for (int j = 0; j < QK8_1; j++) {
- const float v = x[i*QK8_1 + j];
- amax = MAX(amax, fabsf(v));
- }
+ for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = vmaxvq_f32(amaxv[0]);
const float d = amax / ((1 << 7) - 1);
const float id = d ? 1.0f/d : 0.0f;
y[i].d = GGML_FP32_TO_FP16(d);
- int sum = 0;
+ for (int j = 0; j < 8; j++) {
+ const float32x4_t v = vmulq_n_f32(srcv[j], id);
+ const int32x4_t vi = vcvtnq_s32_f32(v);
+
+ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);
+ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);
+ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);
+ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);
+ }
+ }
+#elif defined(__wasm_simd128__)
+ for (int i = 0; i < nb; i++) {
+ v128_t srcv [8];
+ v128_t asrcv[8];
+ v128_t amaxv[8];
+
+ for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);
+ for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);
+
+ for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);
+ for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);
+ for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);
+
+ const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),
+ wasm_f32x4_extract_lane(amaxv[0], 1)),
+ MAX(wasm_f32x4_extract_lane(amaxv[0], 2),
+ wasm_f32x4_extract_lane(amaxv[0], 3)));
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ for (int j = 0; j < 8; j++) {
+ const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));
+ const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);
+
+ y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);
+ y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);
+ y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);
+ y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);
+ }
+ }
+#elif defined(__AVX2__) || defined(__AVX__)
+ for (int i = 0; i < nb; i++) {
+ // Load elements into 4 AVX vectors
+ __m256 v0 = _mm256_loadu_ps( x );
+ __m256 v1 = _mm256_loadu_ps( x + 8 );
+ __m256 v2 = _mm256_loadu_ps( x + 16 );
+ __m256 v3 = _mm256_loadu_ps( x + 24 );
+ x += 32;
+
+ // Compute max(abs(e)) for the block
+ const __m256 signBit = _mm256_set1_ps( -0.0f );
+ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );
+ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );
+
+ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );
+ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );
+ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );
+ const float maxScalar = _mm_cvtss_f32( max4 );
+
+ // Quantize these floats
+ const float d = maxScalar / 127.f;
+ y[i].d = GGML_FP32_TO_FP16(d);
+ const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;
+ const __m256 mul = _mm256_set1_ps( id );
+
+ // Apply the multiplier
+ v0 = _mm256_mul_ps( v0, mul );
+ v1 = _mm256_mul_ps( v1, mul );
+ v2 = _mm256_mul_ps( v2, mul );
+ v3 = _mm256_mul_ps( v3, mul );
+
+ // Round to nearest integer
+ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );
+ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );
+ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );
+ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );
+
+ // Convert floats to integers
+ __m256i i0 = _mm256_cvtps_epi32( v0 );
+ __m256i i1 = _mm256_cvtps_epi32( v1 );
+ __m256i i2 = _mm256_cvtps_epi32( v2 );
+ __m256i i3 = _mm256_cvtps_epi32( v3 );
+
+#if defined(__AVX2__)
+ // Convert int32 to int16
+ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15
+ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31
+ // Convert int16 to int8
+ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
+
+ // We got our precious signed bytes, but the order is now wrong
+ // These AVX2 pack instructions process 16-byte pieces independently
+ // The following instruction is fixing the order
+ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );
+ i0 = _mm256_permutevar8x32_epi32( i0, perm );
+
+ _mm256_storeu_si256((__m256i *)y[i].qs, i0);
+#else
+ // Since we don't have in AVX some necessary functions,
+ // we split the registers in half and call AVX2 analogs from SSE
+ __m128i ni0 = _mm256_castsi256_si128( i0 );
+ __m128i ni1 = _mm256_extractf128_si256( i0, 1);
+ __m128i ni2 = _mm256_castsi256_si128( i1 );
+ __m128i ni3 = _mm256_extractf128_si256( i1, 1);
+ __m128i ni4 = _mm256_castsi256_si128( i2 );
+ __m128i ni5 = _mm256_extractf128_si256( i2, 1);
+ __m128i ni6 = _mm256_castsi256_si128( i3 );
+ __m128i ni7 = _mm256_extractf128_si256( i3, 1);
+
+ // Convert int32 to int16
+ ni0 = _mm_packs_epi32( ni0, ni1 );
+ ni2 = _mm_packs_epi32( ni2, ni3 );
+ ni4 = _mm_packs_epi32( ni4, ni5 );
+ ni6 = _mm_packs_epi32( ni6, ni7 );
+ // Convert int16 to int8
+ ni0 = _mm_packs_epi16( ni0, ni2 );
+ ni4 = _mm_packs_epi16( ni4, ni6 );
+
+ _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
+ _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
+#endif
+ }
+#elif defined(__riscv_v_intrinsic)
+
+ size_t vl = __riscv_vsetvl_e32m4(QK8_0);
+
+ for (int i = 0; i < nb; i++) {
+ // load elements
+ vfloat32m4_t v_x = __riscv_vle32_v_f32m4(x+i*QK8_0, vl);
+
+ vfloat32m4_t vfabs = __riscv_vfabs_v_f32m4(v_x, vl);
+ vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl);
+ vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m4_f32m1(vfabs, tmp, vl);
+ float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);
+
+ // convert to integer
+ vint16m2_t vi = __riscv_vfncvt_x_f_w_i16m2(x0, vl);
+ vint8m1_t vs = __riscv_vncvt_x_x_w_i8m1(vi, vl);
+
+ // store result
+ __riscv_vse8_v_i8m1(y[i].qs , vs, vl);
+ }
+#else
+ GGML_UNUSED(nb);
+ // scalar
+ quantize_row_q8_0_reference(x, y, k);
+#endif
+}
+
+// reference implementation for deterministic creation of model files
+void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict y, int64_t k) {
+ assert(QK8_1 == 32);
+ assert(k % QK8_1 == 0);
+ const int nb = k / QK8_1;
+
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f; // absolute max
+
+ for (int j = 0; j < QK8_1; j++) {
+ const float v = x[i*QK8_1 + j];
+ amax = MAX(amax, fabsf(v));
+ }
+
+ const float d = amax / ((1 << 7) - 1);
+ const float id = d ? 1.0f/d : 0.0f;
+
+ y[i].d = GGML_FP32_TO_FP16(d);
+
+ int sum = 0;
for (int j = 0; j < QK8_1/2; ++j) {
const float v0 = x[i*QK8_1 + j]*id;
@@ -950,6 +1017,8 @@ void dequantize_row_q8_0(const block_q8_0 * restrict x, float * restrict y, int6
// ===================== Helper functions
//
static inline int nearest_int(float fval) {
+ // [jart] https://github.com/ggerganov/llama.cpp/issues/2982
+ // assert(fval <= 4194303.f);
fval = fminf(fval, 4194303.f);
float val = fval + 12582912.f;
int i; memcpy(&i, &val, sizeof(int));
@@ -1198,8 +1267,8 @@ static float make_qkx2_quants(int n, int nmax, const float * restrict x, const f
for (int i = 0; i < n; ++i) {
L[i] = Laux[i];
}
- best_mad = mad;
- scale = this_scale;
+ best_mad = mad;
+ scale = this_scale;
min = this_min;
}
}
@@ -3321,49 +3390,8 @@ static inline __m128i get_scale_shuffle(int i) {
}
#endif
-#ifdef __x86_64__
-#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_default
-#include "ggml_vec_dot_q4_0_q8_0.inc"
-#undef ggml_vec_dot_q4_0_q8_0
-#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q4_0_q8_0_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "ggml_vec_dot_q4_0_q8_0.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef ggml_vec_dot_q4_0_q8_0
-#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_0_q8_0_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "ggml_vec_dot_q4_0_q8_0.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef ggml_vec_dot_q4_0_q8_0
-void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {
- ggml_vec_dot_q4_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);
- } else if (X86_HAVE(AVX)) {
- ggml_vec_dot_q4_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);
- } else {
- ggml_vec_dot_q4_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);
- }
-}
-#else
-#define static
-#include "ggml_vec_dot_q4_0_q8_0.inc"
-#undef static
-#endif
-
-void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- const int qk = QK8_1;
+void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_0;
const int nb = n / qk;
assert(n % qk == 0);
@@ -3377,41 +3405,42 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
UNUSED(by);
UNUSED(bs);
- const block_q4_1 * restrict x = vx;
- const block_q8_1 * restrict y = vy;
+ const block_q4_0 * restrict x = vx;
+ const block_q8_0 * restrict y = vy;
#if defined(__ARM_FEATURE_MATMUL_INT8)
if (nrc == 2) {
- const block_q4_1 * restrict vx0 = vx;
- const block_q4_1 * restrict vx1 = vx + bx;
- const block_q8_1 * restrict vy0 = vy;
- const block_q8_1 * restrict vy1 = vy + by;
+ const block_q4_0 * restrict vx0 = vx;
+ const block_q4_0 * restrict vx1 = vx + bx;
+
+ const block_q8_0 * restrict vy0 = vy;
+ const block_q8_0 * restrict vy1 = vy + by;
float32x4_t sumv0 = vdupq_n_f32(0.0f);
- float32x4_t summs0 = vdupq_n_f32(0.0f);
for (int i = 0; i < nb; i++) {
- const block_q4_1 * restrict b_x0 = &vx0[i];
- const block_q4_1 * restrict b_x1 = &vx1[i];
- const block_q8_1 * restrict b_y0 = &vy0[i];
- const block_q8_1 * restrict b_y1 = &vy1[i];
-
- float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),
- GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),
- GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),
- GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};
- summs0 += summs_t;
+ const block_q4_0 * restrict b_x0 = &vx0[i];
+ const block_q4_0 * restrict b_x1 = &vx1[i];
+ const block_q8_0 * restrict b_y0 = &vy0[i];
+ const block_q8_0 * restrict b_y1 = &vy1[i];
const uint8x16_t m4b = vdupq_n_u8(0x0F);
+ const int8x16_t s8b = vdupq_n_s8(0x8);
const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);
const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);
// 4-bit -> 8-bit
- const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
- const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
- const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
- const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // sub 8
+ const int8x16_t x0_l = vsubq_s8(v0_0l, s8b);
+ const int8x16_t x0_h = vsubq_s8(v0_0h, s8b);
+ const int8x16_t x1_l = vsubq_s8(v0_1l, s8b);
+ const int8x16_t x1_h = vsubq_s8(v0_1h, s8b);
// load y
const int8x16_t y0_l = vld1q_s8(b_y0->qs);
@@ -3419,11 +3448,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
const int8x16_t y1_l = vld1q_s8(b_y1->qs);
const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
- // mmla into int32x4_t
- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,
- GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,
- GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,
- GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};
+ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
@@ -3436,37 +3464,32 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+
sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
l1, r1)), l2, r2)), l3, r3))), scale);
}
-
float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
- sumv2 = sumv2 + summs0;
vst1_f32(s, vget_low_f32(sumv2));
vst1_f32(s + bs, vget_high_f32(sumv2));
return;
}
#endif
- // TODO: add WASM SIMD
#if defined(__ARM_NEON)
float32x4_t sumv0 = vdupq_n_f32(0.0f);
float32x4_t sumv1 = vdupq_n_f32(0.0f);
- float summs = 0;
-
assert(nb % 2 == 0); // TODO: handle odd nb
for (int i = 0; i < nb; i += 2) {
- const block_q4_1 * restrict x0 = &x[i + 0];
- const block_q4_1 * restrict x1 = &x[i + 1];
- const block_q8_1 * restrict y0 = &y[i + 0];
- const block_q8_1 * restrict y1 = &y[i + 1];
-
- summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);
+ const block_q4_0 * restrict x0 = &x[i + 0];
+ const block_q4_0 * restrict x1 = &x[i + 1];
+ const block_q8_0 * restrict y0 = &y[i + 0];
+ const block_q8_0 * restrict y1 = &y[i + 1];
const uint8x16_t m4b = vdupq_n_u8(0x0F);
+ const int8x16_t s8b = vdupq_n_s8(0x8);
const uint8x16_t v0_0 = vld1q_u8(x0->qs);
const uint8x16_t v0_1 = vld1q_u8(x1->qs);
@@ -3477,6 +3500,12 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+ // sub 8
+ const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);
+ const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);
+ const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);
+ const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);
+
// load y
const int8x16_t v1_0l = vld1q_s8(y0->qs);
const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
@@ -3484,56 +3513,192 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
// dot product into int32x4_t
- const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);
- const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);
+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);
+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);
sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
}
- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;
-#elif defined(__AVX2__) || defined(__AVX__)
+ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
+#elif defined(__AVX2__)
// Initialize accumulator with zeros
__m256 acc = _mm256_setzero_ps();
- float summs = 0;
-
// Main loop
for (int i = 0; i < nb; ++i) {
- const float d0 = GGML_FP16_TO_FP32(x[i].d);
- const float d1 = GGML_FP16_TO_FP32(y[i].d);
+ /* Compute combined scale for the block */
+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );
- summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);
+ __m256i qx = bytes_from_nibbles_32(x[i].qs);
- const __m256 d0v = _mm256_set1_ps( d0 );
- const __m256 d1v = _mm256_set1_ps( d1 );
+ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.
+ const __m256i off = _mm256_set1_epi8( 8 );
+ qx = _mm256_sub_epi8( qx, off );
- // Compute combined scales
- const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );
+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
- // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes
-#ifdef __AVX2__
- const __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);
-#else
- const __m256i qx = bytes_from_nibbles_32_avx(x[i].qs);
-#endif
- const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[i].qs );
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
-#if defined(__AVX2__)
- const __m256 xy = mul_sum_us8_pairs_float_avx2(qx, qy);
-#else
- const __m256 xy = mul_sum_us8_pairs_float_avx(qx, qy);
-#endif
+ /* Multiply q with scale and accumulate */
+ acc = _mm256_fmadd_ps( d, q, acc );
+ }
- // Accumulate d0*d1*x*y
-#if defined(__AVX2__)
- acc = _mm256_fmadd_ps( d0d1, xy, acc );
-#else
- acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );
-#endif
+ *s = hsum_float_8(acc);
+#elif defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ // Main loop
+ for (int i = 0; i < nb; ++i) {
+ // Compute combined scale for the block
+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );
+
+ const __m128i lowMask = _mm_set1_epi8(0xF);
+ const __m128i off = _mm_set1_epi8(8);
+
+ const __m128i tmp = _mm_loadu_si128((const __m128i *)x[i].qs);
+
+ __m128i bx_0 = _mm_and_si128(lowMask, tmp);
+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);
+ bx_0 = _mm_sub_epi8(bx_0, off);
+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
+
+ bx_0 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4));
+ by_0 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));
+ bx_0 = _mm_sub_epi8(bx_0, off);
+ const __m128i i32_1 = mul_sum_i8_pairs(bx_0, by_0);
+
+ // Convert int32_t to float
+ __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1));
+
+ // Apply the scale, and accumulate
+ acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);
}
- *s = hsum_float_8(acc) + summs;
+ *s = hsum_float_8(acc);
+#elif defined(__SSSE3__)
+ // set constants
+ const __m128i lowMask = _mm_set1_epi8(0xF);
+ const __m128i off = _mm_set1_epi8(8);
+
+ // Initialize accumulator with zeros
+ __m128 acc_0 = _mm_setzero_ps();
+ __m128 acc_1 = _mm_setzero_ps();
+ __m128 acc_2 = _mm_setzero_ps();
+ __m128 acc_3 = _mm_setzero_ps();
+
+ // First round without accumulation
+ {
+ _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0);
+ _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0);
+
+ // Compute combined scale for the block 0 and 1
+ const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) );
+
+ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[0].qs);
+
+ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);
+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[0].qs);
+ bx_0 = _mm_sub_epi8(bx_0, off);
+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
+
+ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));
+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[0].qs + 16));
+ bx_1 = _mm_sub_epi8(bx_1, off);
+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
+
+ _mm_prefetch(&x[1] + sizeof(block_q4_0), _MM_HINT_T0);
+ _mm_prefetch(&y[1] + sizeof(block_q8_0), _MM_HINT_T0);
+
+ // Compute combined scale for the block 2 and 3
+ const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) );
+
+ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[1].qs);
+
+ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);
+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[1].qs);
+ bx_2 = _mm_sub_epi8(bx_2, off);
+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);
+
+ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));
+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[1].qs + 16));
+ bx_3 = _mm_sub_epi8(bx_3, off);
+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);
+
+ // Convert int32_t to float
+ __m128 p0 = _mm_cvtepi32_ps(i32_0);
+ __m128 p1 = _mm_cvtepi32_ps(i32_1);
+ __m128 p2 = _mm_cvtepi32_ps(i32_2);
+ __m128 p3 = _mm_cvtepi32_ps(i32_3);
+
+ // Apply the scale
+ acc_0 = _mm_mul_ps( d_0_1, p0 );
+ acc_1 = _mm_mul_ps( d_0_1, p1 );
+ acc_2 = _mm_mul_ps( d_2_3, p2 );
+ acc_3 = _mm_mul_ps( d_2_3, p3 );
+ }
+
+ assert(nb % 2 == 0); // TODO: handle odd nb
+
+ // Main loop
+ for (int i = 2; i < nb; i+=2) {
+ _mm_prefetch(&x[i] + sizeof(block_q4_0), _MM_HINT_T0);
+ _mm_prefetch(&y[i] + sizeof(block_q8_0), _MM_HINT_T0);
+
+ // Compute combined scale for the block 0 and 1
+ const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );
+
+ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[i].qs);
+
+ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);
+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);
+ bx_0 = _mm_sub_epi8(bx_0, off);
+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
+
+ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));
+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));
+ bx_1 = _mm_sub_epi8(bx_1, off);
+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
+
+ _mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
+ _mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
+
+ // Compute combined scale for the block 2 and 3
+ const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) );
+
+ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[i + 1].qs);
+
+ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);
+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[i + 1].qs);
+ bx_2 = _mm_sub_epi8(bx_2, off);
+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);
+
+ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));
+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[i + 1].qs + 16));
+ bx_3 = _mm_sub_epi8(bx_3, off);
+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);
+
+ // Convert int32_t to float
+ __m128 p0 = _mm_cvtepi32_ps(i32_0);
+ __m128 p1 = _mm_cvtepi32_ps(i32_1);
+ __m128 p2 = _mm_cvtepi32_ps(i32_2);
+ __m128 p3 = _mm_cvtepi32_ps(i32_3);
+
+ // Apply the scale
+ __m128 p0_d = _mm_mul_ps( d_0_1, p0 );
+ __m128 p1_d = _mm_mul_ps( d_0_1, p1 );
+ __m128 p2_d = _mm_mul_ps( d_2_3, p2 );
+ __m128 p3_d = _mm_mul_ps( d_2_3, p3 );
+
+ // Acummulate
+ acc_0 = _mm_add_ps(p0_d, acc_0);
+ acc_1 = _mm_add_ps(p1_d, acc_1);
+ acc_2 = _mm_add_ps(p2_d, acc_2);
+ acc_3 = _mm_add_ps(p3_d, acc_3);
+ }
+
+ *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);
#elif defined(__riscv_v_intrinsic)
float sumf = 0.0;
@@ -3550,8 +3715,12 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
- vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
- vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ // subtract offset
+ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 8, vl);
+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 8, vl);
vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
@@ -3561,9 +3730,9 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2)mul_sum_us8_pairs_float;
- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);
}
*s = sumf;
@@ -3575,119 +3744,124 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
int sumi = 0;
for (int j = 0; j < qk/2; ++j) {
- const int v0 = (x[i].qs[j] & 0x0F);
- const int v1 = (x[i].qs[j] >> 4);
+ const int v0 = (x[i].qs[j] & 0x0F) - 8;
+ const int v1 = (x[i].qs[j] >> 4) - 8;
sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);
}
- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);
}
*s = sumf;
#endif
}
-#ifdef __x86_64__
-#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_default
-#include "ggml_vec_dot_q5_0_q8_0.inc"
-#undef ggml_vec_dot_q5_0_q8_0
-#define ggml_vec_dot_q5_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q5_0_q8_0_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "ggml_vec_dot_q5_0_q8_0.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef ggml_vec_dot_q5_0_q8_0
-#define ggml_vec_dot_q5_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q5_0_q8_0_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "ggml_vec_dot_q5_0_q8_0.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef ggml_vec_dot_q5_0_q8_0
-void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {
- ggml_vec_dot_q5_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);
- } else if (X86_HAVE(AVX)) {
- ggml_vec_dot_q5_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);
- } else {
- ggml_vec_dot_q5_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);
- }
-}
-#else
-#define static
-#include "ggml_vec_dot_q5_0_q8_0.inc"
-#undef static
-#endif
-
-void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
const int qk = QK8_1;
const int nb = n / qk;
assert(n % qk == 0);
- assert(qk == QK5_1);
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ assert((nrc == 2) || (nrc == 1));
+#else
assert(nrc == 1);
+#endif
UNUSED(nrc);
UNUSED(bx);
UNUSED(by);
UNUSED(bs);
- const block_q5_1 * restrict x = vx;
+ const block_q4_1 * restrict x = vx;
const block_q8_1 * restrict y = vy;
-#if defined(__ARM_NEON)
- float32x4_t sumv0 = vdupq_n_f32(0.0f);
- float32x4_t sumv1 = vdupq_n_f32(0.0f);
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ if (nrc == 2) {
+ const block_q4_1 * restrict vx0 = vx;
+ const block_q4_1 * restrict vx1 = vx + bx;
+ const block_q8_1 * restrict vy0 = vy;
+ const block_q8_1 * restrict vy1 = vy + by;
- float summs0 = 0.0f;
- float summs1 = 0.0f;
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t summs0 = vdupq_n_f32(0.0f);
- uint32_t qh0;
- uint32_t qh1;
+ for (int i = 0; i < nb; i++) {
+ const block_q4_1 * restrict b_x0 = &vx0[i];
+ const block_q4_1 * restrict b_x1 = &vx1[i];
+ const block_q8_1 * restrict b_y0 = &vy0[i];
+ const block_q8_1 * restrict b_y1 = &vy1[i];
- uint64_t tmp0[4];
- uint64_t tmp1[4];
+ float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),
+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),
+ GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),
+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};
+ summs0 += summs_t;
- assert(nb % 2 == 0); // TODO: handle odd nb
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
- for (int i = 0; i < nb; i += 2) {
- const block_q5_1 * restrict x0 = &x[i];
- const block_q5_1 * restrict x1 = &x[i + 1];
- const block_q8_1 * restrict y0 = &y[i];
- const block_q8_1 * restrict y1 = &y[i + 1];
+ const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);
- const uint8x16_t m4b = vdupq_n_u8(0x0F);
+ // 4-bit -> 8-bit
+ const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
- summs0 += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);
- summs1 += GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);
+ // load y
+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);
+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);
+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
- // extract the 5th bit via lookup table ((b) << 4)
- memcpy(&qh0, x0->qh, sizeof(qh0));
- memcpy(&qh1, x1->qh, sizeof(qh1));
+ // mmla into int32x4_t
+ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,
+ GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,
+ GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,
+ GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};
- tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];
- tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];
- tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];
- tmp0[3] = table_b2b_0[(qh0 >> 24) ];
+ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
- tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];
- tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];
- tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];
- tmp1[3] = table_b2b_0[(qh1 >> 24) ];
+ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
- const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));
- const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));
- const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));
- const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));
+ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+
+ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
+ l1, r1)), l2, r2)), l3, r3))), scale);
+ }
+
+ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
+ sumv2 = sumv2 + summs0;
+
+ vst1_f32(s, vget_low_f32(sumv2));
+ vst1_f32(s + bs, vget_high_f32(sumv2));
+ return;
+ }
+#endif
+ // TODO: add WASM SIMD
+#if defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ float summs = 0;
+
+ assert(nb % 2 == 0); // TODO: handle odd nb
+
+ for (int i = 0; i < nb; i += 2) {
+ const block_q4_1 * restrict x0 = &x[i + 0];
+ const block_q4_1 * restrict x1 = &x[i + 1];
+ const block_q8_1 * restrict y0 = &y[i + 0];
+ const block_q8_1 * restrict y1 = &y[i + 1];
+
+ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
const uint8x16_t v0_0 = vld1q_u8(x0->qs);
const uint8x16_t v0_1 = vld1q_u8(x1->qs);
@@ -3698,64 +3872,226 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r
const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
- // add high bit
- const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);
- const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);
- const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);
- const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);
-
// load y
const int8x16_t v1_0l = vld1q_s8(y0->qs);
const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
const int8x16_t v1_1l = vld1q_s8(y1->qs);
const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
- ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),
- ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
- ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),
- ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
- }
+ // dot product into int32x4_t
+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);
+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);
- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;
-#elif defined(__wasm_simd128__)
- v128_t sumv = wasm_f32x4_splat(0.0f);
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
- float summs = 0.0f;
+ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;
+#elif defined(__AVX2__) || defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
- uint32_t qh;
- uint64_t tmp[4];
+ float summs = 0;
- // TODO: check if unrolling this is better
+ // Main loop
for (int i = 0; i < nb; ++i) {
- const block_q5_1 * restrict x0 = &x[i];
- const block_q8_1 * restrict y0 = &y[i];
-
- summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);
+ const float d0 = GGML_FP16_TO_FP32(x[i].d);
+ const float d1 = GGML_FP16_TO_FP32(y[i].d);
- const v128_t m4b = wasm_i8x16_splat(0x0F);
+ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);
- // extract the 5th bit
- memcpy(&qh, x0->qh, sizeof(qh));
+ const __m256 d0v = _mm256_set1_ps( d0 );
+ const __m256 d1v = _mm256_set1_ps( d1 );
- tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];
- tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];
- tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];
- tmp[3] = table_b2b_0[(qh >> 24) ];
+ // Compute combined scales
+ const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );
- const v128_t qhl = wasm_v128_load(tmp + 0);
- const v128_t qhh = wasm_v128_load(tmp + 2);
+ // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes
+ const __m256i qx = bytes_from_nibbles_32(x[i].qs);
+ const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[i].qs );
- const v128_t v0 = wasm_v128_load(x0->qs);
+ const __m256 xy = mul_sum_us8_pairs_float(qx, qy);
- // 4-bit -> 8-bit
- const v128_t v0l = wasm_v128_and (v0, m4b);
- const v128_t v0h = wasm_u8x16_shr(v0, 4);
+ // Accumulate d0*d1*x*y
+#if defined(__AVX2__)
+ acc = _mm256_fmadd_ps( d0d1, xy, acc );
+#else
+ acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );
+#endif
+ }
- // add high bit
- const v128_t v0lf = wasm_v128_or(v0l, qhl);
- const v128_t v0hf = wasm_v128_or(v0h, qhh);
+ *s = hsum_float_8(acc) + summs;
+#elif defined(__riscv_v_intrinsic)
+ float sumf = 0.0;
+
+ size_t vl = __riscv_vsetvl_e8m1(qk/2);
+
+ for (int i = 0; i < nb; i++) {
+ // load elements
+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);
+
+ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);
+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);
+
+ // mask and store lower part of x, and then upper part
+ vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
+ vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
+
+ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
+
+ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
+
+ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
+
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ }
+
+ *s = sumf;
+#else
+ // scalar
+ float sumf = 0.0;
+
+ for (int i = 0; i < nb; i++) {
+ int sumi = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const int v0 = (x[i].qs[j] & 0x0F);
+ const int v1 = (x[i].qs[j] >> 4);
+
+ sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);
+ }
+
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ }
+
+ *s = sumf;
+#endif
+}
+
+void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+
+ assert(n % qk == 0);
+ assert(qk == QK5_0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q5_0 * restrict x = vx;
+ const block_q8_0 * restrict y = vy;
+
+#if defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ uint32_t qh0;
+ uint32_t qh1;
+
+ uint64_t tmp0[4];
+ uint64_t tmp1[4];
+
+ assert(nb % 2 == 0); // TODO: handle odd nb
+
+ for (int i = 0; i < nb; i += 2) {
+ const block_q5_0 * restrict x0 = &x[i];
+ const block_q5_0 * restrict x1 = &x[i + 1];
+ const block_q8_0 * restrict y0 = &y[i];
+ const block_q8_0 * restrict y1 = &y[i + 1];
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+
+ // extract the 5th bit via lookup table ((!b) << 4)
+ memcpy(&qh0, x0->qh, sizeof(qh0));
+ memcpy(&qh1, x1->qh, sizeof(qh1));
+
+ tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];
+ tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];
+ tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];
+ tmp0[3] = table_b2b_1[(qh0 >> 24) ];
+
+ tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];
+ tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];
+ tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];
+ tmp1[3] = table_b2b_1[(qh1 >> 24) ];
+
+ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));
+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));
+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));
+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));
+
+ const uint8x16_t v0_0 = vld1q_u8(x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);
+
+ // 4-bit -> 8-bit
+ int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
+ const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);
+ const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);
+ const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);
+ const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);
+
+ // load y
+ const int8x16_t v1_0l = vld1q_s8(y0->qs);
+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
+ const int8x16_t v1_1l = vld1q_s8(y1->qs);
+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
+#elif defined(__wasm_simd128__)
+ v128_t sumv = wasm_f32x4_splat(0.0f);
+
+ uint32_t qh;
+ uint64_t tmp[4];
+
+ // TODO: check if unrolling this is better
+ for (int i = 0; i < nb; ++i) {
+ const block_q5_0 * restrict x0 = &x[i];
+ const block_q8_0 * restrict y0 = &y[i];
+
+ const v128_t m4b = wasm_i8x16_splat(0x0F);
+
+ // extract the 5th bit
+ memcpy(&qh, x0->qh, sizeof(qh));
+
+ tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];
+ tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];
+ tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];
+ tmp[3] = table_b2b_1[(qh >> 24) ];
+
+ const v128_t qhl = wasm_v128_load(tmp + 0);
+ const v128_t qhh = wasm_v128_load(tmp + 2);
+
+ const v128_t v0 = wasm_v128_load(x0->qs);
+
+ // 4-bit -> 8-bit
+ const v128_t v0l = wasm_v128_and (v0, m4b);
+ const v128_t v0h = wasm_u8x16_shr(v0, 4);
+
+ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
+ const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
+ const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
// load y
const v128_t v1l = wasm_v128_load(y0->qs);
@@ -3773,8 +4109,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r
const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);
// dot product
- sumv = wasm_f32x4_add(sumv,
- wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(
+ sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(
+ wasm_i32x4_add(
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),
wasm_i32x4_dot_i16x8(v0lfh, v1lh)),
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
@@ -3783,67 +4119,61 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r
}
*s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
- wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;
+ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);
#elif defined(__AVX2__)
// Initialize accumulator with zeros
__m256 acc = _mm256_setzero_ps();
- float summs = 0.0f;
-
// Main loop
for (int i = 0; i < nb; i++) {
- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));
-
- summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);
+ /* Compute combined scale for the block */
+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
- __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);
- __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);
- bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));
+ __m256i qx = bytes_from_nibbles_32(x[i].qs);
+ __m256i bxhi = bytes_from_bits_32(x[i].qh);
+ bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));
qx = _mm256_or_si256(qx, bxhi);
- const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));
- const __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
- const __m256 q = mul_sum_us8_pairs_float_avx2(qx, qy);
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
- acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);
+ /* Multiply q with scale and accumulate */
+ acc = _mm256_fmadd_ps(d, q, acc);
}
- *s = hsum_float_8(acc) + summs;
+ *s = hsum_float_8(acc);
#elif defined(__AVX__)
// Initialize accumulator with zeros
__m256 acc = _mm256_setzero_ps();
- __m128i mask = _mm_set1_epi8(0x10);
-
- float summs = 0.0f;
+ __m128i mask = _mm_set1_epi8((char)0xF0);
// Main loop
for (int i = 0; i < nb; i++) {
- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));
-
- summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);
+ /* Compute combined scale for the block */
+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
- __m256i bx_0 = bytes_from_nibbles_32_avx(x[i].qs);
- const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);
+ __m256i bx_0 = bytes_from_nibbles_32(x[i].qs);
+ const __m256i bxhi = bytes_from_bits_32(x[i].qh);
__m128i bxhil = _mm256_castsi256_si128(bxhi);
__m128i bxhih = _mm256_extractf128_si256(bxhi, 1);
- bxhil = _mm_and_si128(bxhil, mask);
- bxhih = _mm_and_si128(bxhih, mask);
+ bxhil = _mm_andnot_si128(bxhil, mask);
+ bxhih = _mm_andnot_si128(bxhih, mask);
__m128i bxl = _mm256_castsi256_si128(bx_0);
__m128i bxh = _mm256_extractf128_si256(bx_0, 1);
bxl = _mm_or_si128(bxl, bxhil);
bxh = _mm_or_si128(bxh, bxhih);
bx_0 = MM256_SET_M128I(bxh, bxl);
- const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));
const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs);
- const __m256 q = mul_sum_us8_pairs_float_avx(bx_0, by_0);
+ const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0);
- acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);
+ /* Multiply q with scale and accumulate */
+ acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);
}
- *s = hsum_float_8(acc) + summs;
+ *s = hsum_float_8(acc);
#elif defined(__riscv_v_intrinsic)
float sumf = 0.0;
@@ -3851,30 +4181,30 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r
size_t vl = __riscv_vsetvl_e8m1(qk/2);
- // temporary registers for shift operations
+ // These temporary registers are for masking and shift operations
vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);
- vuint32m2_t vt_2 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);
+ vuint32m2_t vt_2 = __riscv_vsll_vv_u32m2(__riscv_vmv_v_x_u32m2(1, vl), vt_1, vl);
+
+ vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);
+ vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);
for (int i = 0; i < nb; i++) {
memcpy(&qh, x[i].qh, sizeof(uint32_t));
- // load qh
- vuint32m2_t vqh = __riscv_vmv_v_x_u32m2(qh, vl);
-
- // ((qh >> (j + 0)) << 4) & 0x10;
- vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(vqh, vt_1, vl);
+ // ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);
+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(xha_0, vt_1, vl);
vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);
- vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(xhl_0, 0x10, vl);
- // ((qh >> (j + 12)) ) & 0x10;
- vuint32m2_t xhr_1 = __riscv_vsrl_vv_u32m2(vqh, vt_2, vl);
- vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(xhr_1, 0x10, vl);
+ // ((qh & (1u << (j + 16))) >> (j + 12));
+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(vt_3, qh, vl);
+ vuint32m2_t xhl_1 = __riscv_vsrl_vv_u32m2(xha_1, vt_4, vl);
// narrowing
- vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xha_0, vl);
+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xhl_0, vl);
vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);
- vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xha_1, vl);
+ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xhl_1, vl);
vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);
// load
@@ -3889,8 +4219,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r
vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);
vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);
- vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
- vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 16, vl);
+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 16, vl);
vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
@@ -3902,7 +4235,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r
int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;
}
*s = sumf;
@@ -3917,395 +4250,2306 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r
int sumi = 0;
for (int j = 0; j < qk/2; ++j) {
- const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
- const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
+ const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
+ const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));
- const int32_t x0 = (x[i].qs[j] & 0xF) | xh_0;
- const int32_t x1 = (x[i].qs[j] >> 4) | xh_1;
+ const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;
+ const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;
sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);
}
- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;
}
*s = sumf;
#endif
}
-#ifdef __x86_64__
-#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_default
-#include "ggml_vec_dot_q8_0_q8_0.inc"
-#undef ggml_vec_dot_q8_0_q8_0
-#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q8_0_q8_0_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "ggml_vec_dot_q8_0_q8_0.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef ggml_vec_dot_q8_0_q8_0
-#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q8_0_q8_0_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "ggml_vec_dot_q8_0_q8_0.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef ggml_vec_dot_q8_0_q8_0
-void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {
- ggml_vec_dot_q8_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);
- } else if (X86_HAVE(AVX)) {
- ggml_vec_dot_q8_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);
- } else {
- ggml_vec_dot_q8_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);
- }
-}
-#else
-#define static
-#include "ggml_vec_dot_q8_0_q8_0.inc"
-#undef static
-#endif
+void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_1;
+ const int nb = n / qk;
-#if QK_K == 256
-void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % qk == 0);
+ assert(qk == QK5_1);
assert(nrc == 1);
UNUSED(nrc);
UNUSED(bx);
UNUSED(by);
UNUSED(bs);
- const block_q2_K * restrict x = vx;
- const block_q8_K * restrict y = vy;
+ const block_q5_1 * restrict x = vx;
+ const block_q8_1 * restrict y = vy;
- const int nb = n / QK_K;
+#if defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
-#ifdef __ARM_NEON
- const uint8x16_t m3 = vdupq_n_u8(0x3);
- const uint8x16_t m4 = vdupq_n_u8(0xF);
+ float summs0 = 0.0f;
+ float summs1 = 0.0f;
- const int32x4_t vzero = vdupq_n_s32(0);
+ uint32_t qh0;
+ uint32_t qh1;
- ggml_int8x16x2_t q2bytes;
- uint8_t aux[16];
+ uint64_t tmp0[4];
+ uint64_t tmp1[4];
- float sum = 0;
+ assert(nb % 2 == 0); // TODO: handle odd nb
- for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+ for (int i = 0; i < nb; i += 2) {
+ const block_q5_1 * restrict x0 = &x[i];
+ const block_q5_1 * restrict x1 = &x[i + 1];
+ const block_q8_1 * restrict y0 = &y[i];
+ const block_q8_1 * restrict y1 = &y[i + 1];
- const uint8_t * restrict q2 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
- const uint8_t * restrict sc = x[i].scales;
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
- const uint8x16_t mins_and_scales = vld1q_u8(sc);
- const uint8x16_t scales = vandq_u8(mins_and_scales, m4);
- vst1q_u8(aux, scales);
+ summs0 += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);
+ summs1 += GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);
- const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);
- const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);
- const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};
- const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),
- vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));
- const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),
- vmull_s16(vget_high_s16(mins16.val[1]), vget_high_s16(q8sums.val[1])));
- sum += dmin * vaddvq_s32(vaddq_s32(s0, s1));
+ // extract the 5th bit via lookup table ((b) << 4)
+ memcpy(&qh0, x0->qh, sizeof(qh0));
+ memcpy(&qh1, x1->qh, sizeof(qh1));
- int isum = 0;
- int is = 0;
+ tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];
+ tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];
+ tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];
+ tmp0[3] = table_b2b_0[(qh0 >> 24) ];
+
+ tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];
+ tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];
+ tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];
+ tmp1[3] = table_b2b_0[(qh1 >> 24) ];
+
+ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));
+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));
+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));
+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));
+
+ const uint8x16_t v0_0 = vld1q_u8(x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);
+
+ // 4-bit -> 8-bit
+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // add high bit
+ const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);
+ const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);
+ const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);
+ const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);
+
+ // load y
+ const int8x16_t v1_0l = vld1q_s8(y0->qs);
+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
+ const int8x16_t v1_1l = vld1q_s8(y1->qs);
+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),
+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;
+#elif defined(__wasm_simd128__)
+ v128_t sumv = wasm_f32x4_splat(0.0f);
+
+ float summs = 0.0f;
+
+ uint32_t qh;
+ uint64_t tmp[4];
+
+ // TODO: check if unrolling this is better
+ for (int i = 0; i < nb; ++i) {
+ const block_q5_1 * restrict x0 = &x[i];
+ const block_q8_1 * restrict y0 = &y[i];
+
+ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s);
+
+ const v128_t m4b = wasm_i8x16_splat(0x0F);
+
+ // extract the 5th bit
+ memcpy(&qh, x0->qh, sizeof(qh));
+
+ tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];
+ tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];
+ tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];
+ tmp[3] = table_b2b_0[(qh >> 24) ];
+
+ const v128_t qhl = wasm_v128_load(tmp + 0);
+ const v128_t qhh = wasm_v128_load(tmp + 2);
+
+ const v128_t v0 = wasm_v128_load(x0->qs);
+
+ // 4-bit -> 8-bit
+ const v128_t v0l = wasm_v128_and (v0, m4b);
+ const v128_t v0h = wasm_u8x16_shr(v0, 4);
+
+ // add high bit
+ const v128_t v0lf = wasm_v128_or(v0l, qhl);
+ const v128_t v0hf = wasm_v128_or(v0h, qhh);
+
+ // load y
+ const v128_t v1l = wasm_v128_load(y0->qs);
+ const v128_t v1h = wasm_v128_load(y0->qs + 16);
+
+ // int8x16 -> int16x8
+ const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
+ const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
+ const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);
+ const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);
+
+ const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);
+ const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);
+ const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);
+ const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);
+
+ // dot product
+ sumv = wasm_f32x4_add(sumv,
+ wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),
+ wasm_i32x4_dot_i16x8(v0lfh, v1lh)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
+ wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
+ wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));
+ }
+
+ *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
+ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;
+#elif defined(__AVX2__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ float summs = 0.0f;
+
+ // Main loop
+ for (int i = 0; i < nb; i++) {
+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));
+
+ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);
+
+ __m256i qx = bytes_from_nibbles_32(x[i].qs);
+ __m256i bxhi = bytes_from_bits_32(x[i].qh);
+ bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));
+ qx = _mm256_or_si256(qx, bxhi);
+
+ const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));
+ const __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
+
+ const __m256 q = mul_sum_us8_pairs_float(qx, qy);
+
+ acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);
+ }
+
+ *s = hsum_float_8(acc) + summs;
+#elif defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+ __m128i mask = _mm_set1_epi8(0x10);
+
+ float summs = 0.0f;
+
+ // Main loop
+ for (int i = 0; i < nb; i++) {
+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));
+
+ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);
+
+ __m256i bx_0 = bytes_from_nibbles_32(x[i].qs);
+ const __m256i bxhi = bytes_from_bits_32(x[i].qh);
+ __m128i bxhil = _mm256_castsi256_si128(bxhi);
+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);
+ bxhil = _mm_and_si128(bxhil, mask);
+ bxhih = _mm_and_si128(bxhih, mask);
+ __m128i bxl = _mm256_castsi256_si128(bx_0);
+ __m128i bxh = _mm256_extractf128_si256(bx_0, 1);
+ bxl = _mm_or_si128(bxl, bxhil);
+ bxh = _mm_or_si128(bxh, bxhih);
+ bx_0 = MM256_SET_M128I(bxh, bxl);
+
+ const __m256 dy = _mm256_set1_ps(GGML_FP16_TO_FP32(y[i].d));
+ const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs);
+
+ const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0);
+
+ acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);
+ }
+
+ *s = hsum_float_8(acc) + summs;
+#elif defined(__riscv_v_intrinsic)
+ float sumf = 0.0;
+
+ uint32_t qh;
+
+ size_t vl = __riscv_vsetvl_e8m1(qk/2);
+
+ // temporary registers for shift operations
+ vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);
+ vuint32m2_t vt_2 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);
+
+ for (int i = 0; i < nb; i++) {
+ memcpy(&qh, x[i].qh, sizeof(uint32_t));
+
+ // load qh
+ vuint32m2_t vqh = __riscv_vmv_v_x_u32m2(qh, vl);
+
+ // ((qh >> (j + 0)) << 4) & 0x10;
+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(vqh, vt_1, vl);
+ vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);
+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(xhl_0, 0x10, vl);
+
+ // ((qh >> (j + 12)) ) & 0x10;
+ vuint32m2_t xhr_1 = __riscv_vsrl_vv_u32m2(vqh, vt_2, vl);
+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(xhr_1, 0x10, vl);
+
+ // narrowing
+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xha_0, vl);
+ vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);
+
+ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xha_1, vl);
+ vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);
+
+ // load
+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);
+
+ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);
+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);
+
+ vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
+ vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
+
+ vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);
+ vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);
+
+ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
+
+ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
+
+ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
+
+ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
+
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ }
+
+ *s = sumf;
+#else
+ // scalar
+ float sumf = 0.0;
+
+ for (int i = 0; i < nb; i++) {
+ uint32_t qh;
+ memcpy(&qh, x[i].qh, sizeof(qh));
+
+ int sumi = 0;
+
+ for (int j = 0; j < qk/2; ++j) {
+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
+
+ const int32_t x0 = (x[i].qs[j] & 0xF) | xh_0;
+ const int32_t x1 = (x[i].qs[j] >> 4) | xh_1;
+
+ sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);
+ }
+
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
+ }
+
+ *s = sumf;
+#endif
+}
+
+void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ const int qk = QK8_0;
+ const int nb = n / qk;
+
+ assert(n % qk == 0);
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ assert((nrc == 2) || (nrc == 1));
+#else
+ assert(nrc == 1);
+#endif
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q8_0 * restrict x = vx;
+ const block_q8_0 * restrict y = vy;
+
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ if (nrc == 2) {
+ const block_q8_0 * restrict vx0 = vx;
+ const block_q8_0 * restrict vx1 = vx + bx;
+ const block_q8_0 * restrict vy0 = vy;
+ const block_q8_0 * restrict vy1 = vy + by;
+
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+
+ for (int i = 0; i < nb; i++) {
+ const block_q8_0 * restrict b_x0 = &vx0[i];
+ const block_q8_0 * restrict b_y0 = &vy0[i];
+
+ const block_q8_0 * restrict b_x1 = &vx1[i];
+ const block_q8_0 * restrict b_y1 = &vy1[i];
+
+ const int8x16_t x0_l = vld1q_s8(b_x0->qs);
+ const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16);
+ const int8x16_t x1_l = vld1q_s8(b_x1->qs);
+ const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16);
+
+ // load y
+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);
+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);
+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
+
+ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
+
+ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+
+ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+
+ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+
+ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+
+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
+ l1, r1)), l2, r2)), l3, r3))), scale);
+ }
+ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
+
+ vst1_f32(s, vget_low_f32(sumv2));
+ vst1_f32(s + bs, vget_high_f32(sumv2));
+ return;
+ }
+#endif
+#if defined(__ARM_NEON)
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t sumv1 = vdupq_n_f32(0.0f);
+
+ assert(nb % 2 == 0); // TODO: handle odd nb
+
+ for (int i = 0; i < nb; i += 2) {
+ const block_q8_0 * restrict x0 = &x[i + 0];
+ const block_q8_0 * restrict x1 = &x[i + 1];
+ const block_q8_0 * restrict y0 = &y[i + 0];
+ const block_q8_0 * restrict y1 = &y[i + 1];
+
+ const int8x16_t x0_0 = vld1q_s8(x0->qs);
+ const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);
+ const int8x16_t x1_0 = vld1q_s8(x1->qs);
+ const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);
+
+ // load y
+ const int8x16_t y0_0 = vld1q_s8(y0->qs);
+ const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);
+ const int8x16_t y1_0 = vld1q_s8(y1->qs);
+ const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);
+
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),
+ ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
+ ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),
+ ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
+ }
+
+ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
+#elif defined(__AVX2__) || defined(__AVX__)
+ // Initialize accumulator with zeros
+ __m256 acc = _mm256_setzero_ps();
+
+ // Main loop
+ for (int i = 0; i < nb; ++i) {
+ // Compute combined scale for the block
+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
+ __m256i qx = _mm256_loadu_si256((const __m256i *)x[i].qs);
+ __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
+
+ const __m256 q = mul_sum_i8_pairs_float(qx, qy);
+
+ // Multiply q with scale and accumulate
+#if defined(__AVX2__)
+ acc = _mm256_fmadd_ps( d, q, acc );
+#else
+ acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );
+#endif
+ }
+
+ *s = hsum_float_8(acc);
+#elif defined(__riscv_v_intrinsic)
+ float sumf = 0.0;
+ size_t vl = __riscv_vsetvl_e8m1(qk);
+
+ for (int i = 0; i < nb; i++) {
+ // load elements
+ vint8m1_t bx_0 = __riscv_vle8_v_i8m1(x[i].qs, vl);
+ vint8m1_t by_0 = __riscv_vle8_v_i8m1(y[i].qs, vl);
+
+ vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx_0, by_0, vl);
+
+ vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);
+ vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl);
+
+ int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);
+
+ sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));
+ }
+
+ *s = sumf;
+#else
+ // scalar
+ float sumf = 0.0;
+
+ for (int i = 0; i < nb; i++) {
+ int sumi = 0;
+
+ for (int j = 0; j < qk; j++) {
+ sumi += x[i].qs[j]*y[i].qs[j];
+ }
+
+ sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));
+ }
+
+ *s = sumf;
+#endif
+}
+
+#if QK_K == 256
+void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q2_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#ifdef __ARM_NEON
+ const uint8x16_t m3 = vdupq_n_u8(0x3);
+ const uint8x16_t m4 = vdupq_n_u8(0xF);
+
+ const int32x4_t vzero = vdupq_n_s32(0);
+
+ ggml_int8x16x2_t q2bytes;
+ uint8_t aux[16];
+
+ float sum = 0;
+
+ for (int i = 0; i < nb; ++i) {
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ const uint8_t * restrict sc = x[i].scales;
+
+ const uint8x16_t mins_and_scales = vld1q_u8(sc);
+ const uint8x16_t scales = vandq_u8(mins_and_scales, m4);
+ vst1q_u8(aux, scales);
+
+ const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);
+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);
+ const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};
+ const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),
+ vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));
+ const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),
+ vmull_s16(vget_high_s16(mins16.val[1]), vget_high_s16(q8sums.val[1])));
+ sum += dmin * vaddvq_s32(vaddq_s32(s0, s1));
+
+ int isum = 0;
+ int is = 0;
+
+// We use this macro instead of a function call because for some reason
+// the code runs 2-3% slower, even if the function is declared inline
+#define MULTIPLY_ACCUM_WITH_SCALE(index)\
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];
+
+#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\
+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\
+ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\
+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\
+ MULTIPLY_ACCUM_WITH_SCALE((index));
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;
+
+ ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
+ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));
+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));
+
+ MULTIPLY_ACCUM_WITH_SCALE(0);
+
+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);
+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);
+ SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);
+
+ is += 8;
+ }
+
+ sum += d * isum;
+ }
+
+ *s = sum;
+
+#elif defined __AVX2__
+
+ const __m256i m3 = _mm256_set1_epi8(3);
+ const __m128i m4 = _mm_set1_epi8(0xF);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);
+ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);
+ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);
+ const __m256i mins = _mm256_cvtepi8_epi16(mins8);
+ const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));
+
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);
+
+ const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);
+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);
+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);
+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};
+
+ __m256i sumi = _mm256_setzero_si256();
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;
+
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ const __m256i q2_0 = _mm256_and_si256(q2bits, m3);
+ const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);
+ const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);
+ const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);
+
+ __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);
+ __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);
+ __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);
+ __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);
+
+ p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);
+ p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);
+ p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);
+ p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);
+
+ p0 = _mm256_add_epi32(p0, p1);
+ p2 = _mm256_add_epi32(p2, p3);
+
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));
+ }
+
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __AVX__
+
+ const __m128i m3 = _mm_set1_epi8(0x3);
+ const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m128i m2 = _mm_set1_epi8(0x2);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ // load mins and scales from block_q2_K.scales[QK_K/16]
+ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);
+ const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);
+ const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);
+ const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);
+ const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));
+
+ // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2
+ const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));
+ const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));
+
+ // sumf += -dmin * summs in 32bits*8
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);
+
+ const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);
+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));
+ const __m128i scales[2] = { scales_0, scales_1 };
+
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]
+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+
+ // load 2bits*16*8 from block_q2_K.qs[QK_K/4]
+ __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;
+ const __m128i q2_0 = _mm_and_si128(q2bits, m3);
+ const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
+ const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
+ const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
+ q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;
+ const __m128i q2_1 = _mm_and_si128(q2bits, m3);
+ const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
+ const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
+ const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
+
+ // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8
+ __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);
+ __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);
+ __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);
+ __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);
+ __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);
+ __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);
+ __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);
+ __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);
+
+ // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);
+
+ p0 = _mm_add_epi32(p0, p1);
+ p2 = _mm_add_epi32(p2, p3);
+ p4 = _mm_add_epi32(p4, p5);
+ p6 = _mm_add_epi32(p6, p7);
+
+ // isum in 32bits*4*2
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));
+ }
+
+ // sumf += dall * isum - dmin * summs in 32bits
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __riscv_v_intrinsic
+
+ float sumf = 0;
+ uint8_t temp_01[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * q2 = x[i].qs;
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * sc = x[i].scales;
+
+ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ size_t vl = 16;
+
+ vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl);
+ vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl);
+
+ vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl);
+
+ vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl);
+ vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl);
+ vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));
+ vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl);
+ vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
+
+ sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums);
+
+ vl = 32;
+
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+ vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl);
+
+ uint8_t is=0;
+ int isum=0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ // load Q2
+ vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl);
+
+ vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl);
+ vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03 , vl);
+ vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03 , vl);
+ vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03 , vl);
+
+ // duplicate scale elements for product
+ vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0+is, vl), vl);
+ vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2+is, vl), vl);
+ vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4+is, vl), vl);
+ vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6+is, vl), vl);
+
+ vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl));
+ vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl));
+ vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl));
+ vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl));
+
+ // load Q8
+ vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);
+ vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);
+ vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8+64, vl);
+ vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8+96, vl);
+
+ vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl);
+ vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl);
+ vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl);
+ vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl);
+
+ vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl);
+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl);
+
+ isum += __riscv_vmv_x_s_i32m1_i32(isum1);
+
+ q2+=32; q8+=128; is=8;
+
+ }
+
+ sumf += dall * isum;
+
+ }
+
+ *s = sumf;
+
+#else
+
+ float sumf = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * q2 = x[i].qs;
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * sc = x[i].scales;
+
+ int summs = 0;
+ for (int j = 0; j < 16; ++j) {
+ summs += y[i].bsums[j] * (sc[j] >> 4);
+ }
+
+ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ int isum = 0;
+ int is = 0;
+ int d;
+ for (int k = 0; k < QK_K/128; ++k) {
+ int shift = 0;
+ for (int j = 0; j < 4; ++j) {
+ d = sc[is++] & 0xF;
+ int isuml = 0;
+ for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
+ isum += d * isuml;
+ d = sc[is++] & 0xF;
+ isuml = 0;
+ for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
+ isum += d * isuml;
+ shift += 2;
+ q8 += 32;
+ }
+ q2 += 32;
+ }
+ sumf += dall * isum - dmin * summs;
+ }
+ *s = sumf;
+#endif
+}
+
+#else
+
+void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q2_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#ifdef __ARM_NEON
+ const uint8x16_t m3 = vdupq_n_u8(0x3);
+
+ const int32x4_t vzero = vdupq_n_s32(0);
+
+ ggml_int8x16x4_t q2bytes;
+
+ uint32_t aux32[2];
+ const uint8_t * scales = (const uint8_t *)aux32;
+
+ float sum = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
+
+ aux32[0] = sc[0] & 0x0f0f0f0f;
+ aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;
+
+ sum += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);
+
+ int isum1 = 0, isum2 = 0;
+
+ const uint8x16_t q2bits = vld1q_u8(q2);
+
+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);
+
+ q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits, m3));
+ q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 2), m3));
+ q2bytes.val[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 4), m3));
+ q2bytes.val[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 6), m3));
+
+ isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * scales[0];
+ isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * scales[1];
+ isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[2], q8bytes.val[2])) * scales[2];
+ isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[3], q8bytes.val[3])) * scales[3];
+
+ sum += d * (isum1 + isum2);
+ }
+
+ *s = sum;
+
+#elif defined __AVX2__
+
+ const __m256i m3 = _mm256_set1_epi8(3);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ uint32_t ud, um;
+ const uint8_t * restrict db = (const uint8_t *)&ud;
+ const uint8_t * restrict mb = (const uint8_t *)&um;
+
+ float summs = 0;
+
+ // TODO: optimize this
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
+ ud = (sc[0] >> 0) & 0x0f0f0f0f;
+ um = (sc[0] >> 4) & 0x0f0f0f0f;
+
+ int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];
+ summs += dmin * smin;
+
+ const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);
+ const __m256i q2_0 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 2), q2bits), m3);
+ const __m256i q2_1 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 6), _mm_srli_epi16(q2bits, 4)), m3);
+
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+
+ const __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);
+ const __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);
+
+ const __m256i p_0 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 0));
+ const __m256i p_1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 1));
+ const __m256i p_2 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 0));
+ const __m256i p_3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 1));
+
+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0), acc);
+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1), acc);
+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2), acc);
+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3), acc);
+ }
+
+ *s = hsum_float_8(acc) + summs;
+
+#elif defined __AVX__
+
+ const __m128i m3 = _mm_set1_epi8(3);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ uint32_t ud, um;
+ const uint8_t * restrict db = (const uint8_t *)&ud;
+ const uint8_t * restrict mb = (const uint8_t *)&um;
+
+ float summs = 0;
+
+ // TODO: optimize this
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
+ ud = (sc[0] >> 0) & 0x0f0f0f0f;
+ um = (sc[0] >> 4) & 0x0f0f0f0f;
+
+ int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];
+ summs += dmin * smin;
+
+ const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);
+ const __m128i q2_0 = _mm_and_si128(q2bits, m3);
+ const __m128i q2_1 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
+ const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
+ const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
+
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+
+ const __m128i p0 = _mm_maddubs_epi16(q2_0, _mm256_extractf128_si256(q8_0, 0));
+ const __m128i p1 = _mm_maddubs_epi16(q2_1, _mm256_extractf128_si256(q8_0, 1));
+ const __m128i p2 = _mm_maddubs_epi16(q2_2, _mm256_extractf128_si256(q8_1, 0));
+ const __m128i p3 = _mm_maddubs_epi16(q2_3, _mm256_extractf128_si256(q8_1, 1));
+
+ const __m256i p_0 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p0, p0)), _mm_cvtepi16_epi32(p0));
+ const __m256i p_1 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p1, p1)), _mm_cvtepi16_epi32(p1));
+ const __m256i p_2 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p2, p2)), _mm_cvtepi16_epi32(p2));
+ const __m256i p_3 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p3, p3)), _mm_cvtepi16_epi32(p3));
+
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0)), acc);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1)), acc);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2)), acc);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3)), acc);
+ }
+
+ *s = hsum_float_8(acc) + summs;
+
+#elif defined __riscv_v_intrinsic
+
+ uint32_t aux32[2];
+ const uint8_t * scales = (const uint8_t *)aux32;
+
+ float sumf = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const uint8_t * restrict q2 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
+
+ aux32[0] = sc[0] & 0x0f0f0f0f;
+ aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;
+
+ sumf += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);
+
+ int isum1 = 0;
+ int isum2 = 0;
+
+ size_t vl = 16;
+
+ vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);
+
+ // load Q2
+ vuint8mf2_t q2_x = __riscv_vle8_v_u8mf2(q2, vl);
+
+ vint8mf2_t q2_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q2_x, 0x03, vl));
+ vint8mf2_t q2_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x2, vl), 0x03 , vl));
+ vint8mf2_t q2_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x4, vl), 0x03 , vl));
+ vint8mf2_t q2_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x6, vl), 0x03 , vl));
+
+ // load Q8, and take product with Q2
+ vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q2_0, __riscv_vle8_v_i8mf2(q8, vl), vl);
+ vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q2_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);
+ vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q2_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);
+ vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q2_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);
+
+ vint16m1_t vs_0 = __riscv_vredsum_vs_i16m1_i16m1(p0, vzero, vl);
+ vint16m1_t vs_1 = __riscv_vredsum_vs_i16m1_i16m1(p1, vzero, vl);
+ vint16m1_t vs_2 = __riscv_vredsum_vs_i16m1_i16m1(p2, vzero, vl);
+ vint16m1_t vs_3 = __riscv_vredsum_vs_i16m1_i16m1(p3, vzero, vl);
+
+ isum1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[0];
+ isum2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[1];
+ isum1 += __riscv_vmv_x_s_i16m1_i16(vs_2) * scales[2];
+ isum2 += __riscv_vmv_x_s_i16m1_i16(vs_3) * scales[3];
+
+ sumf += d * (isum1 + isum2);
+
+ }
+
+ *s = sumf;
+
+#else
+
+ float sumf = 0;
+
+ int isum[QK_K/16];
+
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * q2 = x[i].qs;
+ const int8_t * q8 = y[i].qs;
+ const uint8_t * sc = x[i].scales;
+
+ int summs = 0;
+ for (int j = 0; j < QK_K/16; ++j) {
+ summs += y[i].bsums[j] * (sc[j] >> 4);
+ }
+
+ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ memset(isum, 0, (QK_K/16)*sizeof(int));
+ for (int l = 0; l < 16; ++l) {
+ isum[0] += q8[l+ 0] * ((q2[l] >> 0) & 3);
+ isum[1] += q8[l+16] * ((q2[l] >> 2) & 3);
+ isum[2] += q8[l+32] * ((q2[l] >> 4) & 3);
+ isum[3] += q8[l+48] * ((q2[l] >> 6) & 3);
+ }
+ for (int l = 0; l < QK_K/16; ++l) {
+ isum[l] *= (sc[l] & 0xF);
+ }
+ sumf += dall * (isum[0] + isum[1] + isum[2] + isum[3]) - dmin * summs;
+ }
+ *s = sumf;
+#endif
+}
+#endif
+
+#if QK_K == 256
+void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const uint32_t kmask1 = 0x03030303;
+ const uint32_t kmask2 = 0x0f0f0f0f;
+
+ const block_q3_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#ifdef __ARM_NEON
+
+ uint32_t aux[3];
+ uint32_t utmp[4];
+
+ const uint8x16_t m3b = vdupq_n_u8(0x3);
+ const int32x4_t vzero = vdupq_n_s32(0);
+
+ const uint8x16_t m0 = vdupq_n_u8(1);
+ const uint8x16_t m1 = vshlq_n_u8(m0, 1);
+ const uint8x16_t m2 = vshlq_n_u8(m0, 2);
+ const uint8x16_t m3 = vshlq_n_u8(m0, 3);
+ const int8_t m32 = 32;
+
+ ggml_int8x16x4_t q3bytes;
+
+ float sum = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict qh = x[i].hmask;
+ const int8_t * restrict q8 = y[i].qs;
+
+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);
+
+ ggml_uint8x16x4_t q3h;
+
+ int32_t isum = 0;
+
+ // Set up scales
+ memcpy(aux, x[i].scales, 12);
+ utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);
+ utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);
+ utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);
+ utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);
+
+ int8_t * scale = (int8_t *)utmp;
+ for (int j = 0; j < 16; ++j) scale[j] -= m32;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+
+ const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;
+ const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;
+ const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);
+ q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);
+ q3h.val[2] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[0]), 1);
+ q3h.val[3] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[1]), 1);
+
+ q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[0], m3b)), vreinterpretq_s8_u8(q3h.val[0]));
+ q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[1], m3b)), vreinterpretq_s8_u8(q3h.val[1]));
+ q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));
+ q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];
+
+ scale += 4;
+
+ q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);
+ q3h.val[1] = vbicq_u8(m2, qhbits.val[1]);
+ q3h.val[2] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[0]), 1);
+ q3h.val[3] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[1]), 1);
+
+ q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 4), m3b)), vreinterpretq_s8_u8(q3h.val[0]));
+ q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 4), m3b)), vreinterpretq_s8_u8(q3h.val[1]));
+ q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));
+ q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];
+
+ scale += 4;
+
+ if (j == 0) {
+ qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);
+ qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);
+ }
+
+ }
+ sum += d * isum;
+
+ }
+
+ *s = sum;
+
+#elif defined __AVX2__
+
+ const __m256i m3 = _mm256_set1_epi8(3);
+ const __m256i mone = _mm256_set1_epi8(1);
+ const __m128i m32 = _mm_set1_epi8(32);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ uint32_t aux[3];
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ // Set up scales
+ memcpy(aux, x[i].scales, 12);
+ __m128i scales128 = _mm_set_epi32(
+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
+ scales128 = _mm_sub_epi8(scales128, m32);
+ const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);
+ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);
+ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);
+ const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};
+
+ // high bit
+ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);
+
+ // integer accumulator
+ __m256i sumi = _mm256_setzero_si256();
+
+ int bit = 0;
+ int is = 0;
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ // load low 2 bits
+ const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;
+
+ // prepare low and high bits
+ const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);
+ const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);
+ const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);
+ const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);
+ const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
+ ++bit;
+
+ // load Q8 quants
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
+ // and 2 if the high bit was set)
+ __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);
+ __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);
+ __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);
+ __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);
+
+ __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);
+ __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);
+ __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);
+ __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);
+
+ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);
+
+ // multiply with scales
+ p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);
+ p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);
+ p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);
+ p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);
+
+ // accumulate
+ p16_0 = _mm256_add_epi32(p16_0, p16_1);
+ p16_2 = _mm256_add_epi32(p16_2, p16_3);
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));
+
+ }
+
+ // multiply with block scale and accumulate
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __AVX__
+
+ const __m128i m3 = _mm_set1_epi8(3);
+ const __m128i mone = _mm_set1_epi8(1);
+ const __m128i m32 = _mm_set1_epi8(32);
+ const __m128i m2 = _mm_set1_epi8(2);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ const uint32_t *aux;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ // Set up scales
+ aux = (const uint32_t *)x[i].scales;
+ __m128i scales128 = _mm_set_epi32(
+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
+ scales128 = _mm_sub_epi8(scales128, m32);
+ const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);
+ const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));
+ const __m128i scales[2] = { scales_0, scales_1 };
+
+ // high bit *128*2 from block_q3_K.hmask[QK_K/8]
+ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);
+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);
+
+ // integer accumulator
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
+
+ for (int j = 0; j < QK_K/128; ++j) {
+ // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]
+ const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;
+ const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;
+
+ // prepare low and high bits
+ const int bit = j << 2;
+
+ const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);
+ const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);
+ const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);
+ const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);
+
+ const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);
+ const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);
+ const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);
+ const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);
+
+ const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);
+ const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);
+ const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);
+ const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);
+
+ const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);
+ const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);
+ const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);
+ const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);
+
+ // load Q8 quants from block_q8_K.qs[QK_K]
+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+
+ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
+ // and 2 if the high bit was set)
+ __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);
+ __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);
+ __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);
+ __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);
+ __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);
+ __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);
+ __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);
+ __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);
+
+ __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);
+ __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);
+ __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);
+ __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);
+ __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);
+ __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);
+ __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);
+ __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);
+
+ p16_0 = _mm_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);
+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);
+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);
+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);
+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);
+
+ // multiply with scales
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);
+
+ // accumulate
+ p16_0 = _mm_add_epi32(p16_0, p16_1);
+ p16_2 = _mm_add_epi32(p16_2, p16_3);
+ p16_4 = _mm_add_epi32(p16_4, p16_5);
+ p16_6 = _mm_add_epi32(p16_6, p16_7);
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));
+
+ }
+
+ // multiply with block scale and accumulate
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __riscv_v_intrinsic
+
+ uint32_t aux[3];
+ uint32_t utmp[4];
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict qh = x[i].hmask;
+ const int8_t * restrict q8 = y[i].qs;
+
+ memcpy(aux, x[i].scales, 12);
+ utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);
+ utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);
+ utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);
+ utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);
+
+ int8_t * scale = (int8_t *)utmp;
+ for (int j = 0; j < 16; ++j) scale[j] -= 32;
+
+
+ size_t vl = 32;
+ uint8_t m = 1;
+
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+ vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl);
+
+ int sum_t = 0;
+
+ for (int j = 0; j < QK_K; j += 128) {
+
+ vl = 32;
+
+ // load Q3
+ vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl);
+
+ vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl));
+ vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl));
+ vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl));
+ vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl));
+
+ // compute mask for subtraction
+ vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl);
+ vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_m(vmask_0, q3_0, 0x4, vl);
+ m <<= 1;
+
+ vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl);
+ vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_m(vmask_1, q3_1, 0x4, vl);
+ m <<= 1;
+
+ vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl);
+ vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_m(vmask_2, q3_2, 0x4, vl);
+ m <<= 1;
+
+ vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl);
+ vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_m(vmask_3, q3_3, 0x4, vl);
+ m <<= 1;
+
+ // load Q8 and take product with Q3
+ vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl);
+ vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl);
+ vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl);
+ vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl);
+
+ vl = 16;
+
+ // retrieve lane to multiply with scale
+ vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl);
+ vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl);
+ vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl);
+ vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl);
+ vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl);
+ vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl);
+ vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl);
+ vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl);
+
+ vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl);
+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl);
+ vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl);
+ vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl);
+
+ sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);
+
+ q3 += 32; q8 += 128; scale += 8;
+
+ }
+
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+
+ sumf += d*sum_t;
+
+ }
+
+ *s = sumf;
+
+#else
+ // scalar version
+ // This function is written like this so the compiler can manage to vectorize most of it
+ // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the
+ // manually vectorized version above. Every other version I tried would run at least 4 times slower.
+ // The ideal situation would be if we could just write the code once, and the compiler would
+ // automatically produce the best possible set of machine instructions, instead of us having to manually
+ // write vectorized versions for AVX, ARM_NEON, etc.
+
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
+ float sums [8];
+ int32_t aux32[8];
+ memset(sums, 0, 8*sizeof(float));
+
+ uint32_t auxs[4];
+ const int8_t * scales = (const int8_t*)auxs;
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict hm = x[i].hmask;
+ const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
+ int8_t * restrict a = aux8;
+ uint8_t m = 1;
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;
+ for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
+ a += 32; m <<= 1;
+ q3 += 32;
+ }
+ a = aux8;
+
+ memcpy(auxs, x[i].scales, 12);
+ uint32_t tmp = auxs[2];
+ auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
+ auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
+ auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
+ auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
+ for (int j = 0; j < QK_K/16; ++j) {
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
+ q8 += 8; a += 8;
+ }
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ }
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
+ *s = sumf;
+
+#endif
+
+}
+
+#else
+
+void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q3_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
+
+ const int nb = n / QK_K;
+
+#ifdef __ARM_NEON
+ const int32x4_t vzero = vdupq_n_s32(0);
+
+ const uint8x16_t m3b = vdupq_n_u8(0x3);
+ const uint8x16_t mh = vdupq_n_u8(4);
+
+ ggml_int8x16x4_t q3bytes;
+
+ uint16_t aux16[2];
+ int8_t * scales = (int8_t *)aux16;
+
+ float sum = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ ggml_uint8x16x4_t q3h;
+
+ const uint8x8_t hbits = vld1_u8(x[i].hmask);
+ const uint8x16_t q3bits = vld1q_u8(x[i].qs);
+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(y[i].qs);
+
+ const uint16_t a = *(const uint16_t *)x[i].scales;
+ aux16[0] = a & 0x0f0f;
+ aux16[1] = (a >> 4) & 0x0f0f;
+
+ for (int j = 0; j < 4; ++j) scales[j] -= 8;
+
+ int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8x16_t htmp = vcombine_u8(hbits, vshr_n_u8(hbits, 1));
+ q3h.val[0] = vandq_u8(mh, vshlq_n_u8(htmp, 2));
+ q3h.val[1] = vandq_u8(mh, htmp);
+ q3h.val[2] = vandq_u8(mh, vshrq_n_u8(htmp, 2));
+ q3h.val[3] = vandq_u8(mh, vshrq_n_u8(htmp, 4));
+
+ q3bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q3bits, m3b), q3h.val[0]));
+ q3bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 2), m3b), q3h.val[1]));
+ q3bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 4), m3b), q3h.val[2]));
+ q3bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q3bits, 6), q3h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes.val[0])) * scales[0];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes.val[1])) * scales[2];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes.val[2])) * scales[1];
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes.val[3])) * scales[3];
+
+ sum += d * isum;
+
+ }
+
+ *s = sum;
+
+#elif defined __AVX2__
+
+ const __m256i m3 = _mm256_set1_epi8(3);
+ const __m256i m1 = _mm256_set1_epi8(1);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ uint64_t aux64;
+
+ uint16_t aux16[2];
+ const int8_t * aux8 = (const int8_t *)aux16;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const uint16_t a = *(const uint16_t *)x[i].scales;
+ aux16[0] = a & 0x0f0f;
+ aux16[1] = (a >> 4) & 0x0f0f;
+
+ const __m256i scale_0 = MM256_SET_M128I(_mm_set1_epi16(aux8[2] - 8), _mm_set1_epi16(aux8[0] - 8));
+ const __m256i scale_1 = MM256_SET_M128I(_mm_set1_epi16(aux8[3] - 8), _mm_set1_epi16(aux8[1] - 8));
+
+ memcpy(&aux64, x[i].hmask, 8);
+
+ const __m128i haux = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);
+ __m256i q3h_0 = MM256_SET_M128I(_mm_srli_epi16(haux, 2), haux);
+ __m256i q3h_1 = _mm256_srli_epi16(q3h_0, 4);
+ q3h_0 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_0, m1), 2);
+ q3h_1 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_1, m1), 2);
+
+ // load low 2 bits
+ const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);
+
+ // prepare low and high bits
+ const __m256i q3aux = MM256_SET_M128I(_mm_srli_epi16(q3bits, 2), q3bits);
+ const __m256i q3l_0 = _mm256_and_si256(q3aux, m3);
+ const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3aux, 4), m3);
+
+ // load Q8 quants
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+
+ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
+ // and 2 if the high bit was set)
+ const __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);
+ const __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);
+
+ __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);
+ __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);
+
+ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
+
+ // multiply with scales
+ p16_0 = _mm256_madd_epi16(scale_0, p16_0);
+ p16_1 = _mm256_madd_epi16(scale_1, p16_1);
+
+ p16_0 = _mm256_add_epi32(p16_0, p16_1);
+
+ // multiply with block scale and accumulate
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16_0), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __AVX__
+
+ const __m128i m3 = _mm_set1_epi8(3);
+ const __m128i m1 = _mm_set1_epi8(1);
+
+ __m256 acc = _mm256_setzero_ps();
+
+ uint64_t aux64;
+
+ uint16_t aux16[2];
+ const int8_t * aux8 = (const int8_t *)aux16;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const uint16_t a = *(const uint16_t *)x[i].scales;
+ aux16[0] = a & 0x0f0f;
+ aux16[1] = (a >> 4) & 0x0f0f;
+
+ const __m128i scale_0 = _mm_set1_epi16(aux8[0] - 8);
+ const __m128i scale_1 = _mm_set1_epi16(aux8[2] - 8);
+ const __m128i scale_2 = _mm_set1_epi16(aux8[1] - 8);
+ const __m128i scale_3 = _mm_set1_epi16(aux8[3] - 8);
+
+ memcpy(&aux64, x[i].hmask, 8);
+
+ __m128i q3h_0 = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);
+ __m128i q3h_1 = _mm_srli_epi16(q3h_0, 2);
+ __m128i q3h_2 = _mm_srli_epi16(q3h_0, 4);
+ __m128i q3h_3 = _mm_srli_epi16(q3h_0, 6);
+ q3h_0 = _mm_slli_epi16(_mm_andnot_si128(q3h_0, m1), 2);
+ q3h_1 = _mm_slli_epi16(_mm_andnot_si128(q3h_1, m1), 2);
+ q3h_2 = _mm_slli_epi16(_mm_andnot_si128(q3h_2, m1), 2);
+ q3h_3 = _mm_slli_epi16(_mm_andnot_si128(q3h_3, m1), 2);
+
+ // load low 2 bits
+ const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);
+
+ // prepare low and high bits
+ const __m128i q3l_0 = _mm_and_si128(q3bits, m3);
+ const __m128i q3l_1 = _mm_and_si128(_mm_srli_epi16(q3bits, 2), m3);
+ const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits, 4), m3);
+ const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits, 6), m3);
+
+ // load Q8 quants
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+
+ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm_maddubs_epi16,
+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
+ // and 2 if the high bit was set)
+ const __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, _mm256_extractf128_si256(q8_0, 0));
+ const __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, _mm256_extractf128_si256(q8_0, 1));
+ const __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, _mm256_extractf128_si256(q8_1, 0));
+ const __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, _mm256_extractf128_si256(q8_1, 1));
+
+ __m128i p16_0 = _mm_maddubs_epi16(q3l_0, _mm256_extractf128_si256(q8_0, 0));
+ __m128i p16_1 = _mm_maddubs_epi16(q3l_1, _mm256_extractf128_si256(q8_0, 1));
+ __m128i p16_2 = _mm_maddubs_epi16(q3l_2, _mm256_extractf128_si256(q8_1, 0));
+ __m128i p16_3 = _mm_maddubs_epi16(q3l_3, _mm256_extractf128_si256(q8_1, 1));
+
+ p16_0 = _mm_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);
+
+ // multiply with scales
+ p16_0 = _mm_madd_epi16(scale_0, p16_0);
+ p16_1 = _mm_madd_epi16(scale_1, p16_1);
+ p16_2 = _mm_madd_epi16(scale_2, p16_2);
+ p16_3 = _mm_madd_epi16(scale_3, p16_3);
+
+ p16_0 = _mm_add_epi32(p16_0, p16_2);
+ p16_1 = _mm_add_epi32(p16_1, p16_3);
+ __m256i p16 = MM256_SET_M128I(p16_1, p16_0);
+
+ // multiply with block scale and accumulate
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16)), acc);
+
+ }
+
+ *s = hsum_float_8(acc);
+
+#elif defined __riscv_v_intrinsic
+
+ uint16_t aux16[2];
+ int8_t * scales = (int8_t *)aux16;
+
+ float sumf = 0;
+
+ for (int i = 0; i < nb; ++i) {
+
+ const uint8_t * restrict q3 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const uint16_t a = *(const uint16_t *)x[i].scales;
+ aux16[0] = a & 0x0f0f;
+ aux16[1] = (a >> 4) & 0x0f0f;
+
+ for (int j = 0; j < 4; ++j) scales[j] -= 8;
+
+ int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+
+ // load qh
+ vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(x[i].hmask, 8);
+ vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));
+
+ size_t vl = 16;
+
+ // extend and combine both qh_x1 and qh_x2
+ vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);
+
+ vuint8mf2_t qh_0 = __riscv_vand_vx_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);
+ vuint8mf2_t qh_1 = __riscv_vand_vx_u8mf2(qh_x, 0x4, vl);
+ vuint8mf2_t qh_2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);
+ vuint8mf2_t qh_3 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), 0x4, vl);
+
+ // load Q3
+ vuint8mf2_t q3_x = __riscv_vle8_v_u8mf2(q3, vl);
+
+ vuint8mf2_t q3h_0 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(q3_x, 0x3, vl), qh_0, vl);
+ vuint8mf2_t q3h_1 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 2, vl), 0x3, vl), qh_1, vl);
+ vuint8mf2_t q3h_2 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 4, vl), 0x3, vl), qh_2, vl);
+ vuint8mf2_t q3h_3 = __riscv_vor_vv_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x6, vl), qh_3, vl);
+
+ vint8mf2_t q3_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_0);
+ vint8mf2_t q3_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_1);
+ vint8mf2_t q3_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_2);
+ vint8mf2_t q3_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_3);
+
+ // load Q8 and take product with Q3
+ vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q3_0, __riscv_vle8_v_i8mf2(q8, vl), vl);
+ vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q3_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);
+ vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q3_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);
+ vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q3_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);
+
+ vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);
+ vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);
+ vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);
+ vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);
+
+ isum += __riscv_vmv_x_s_i32m1_i32(vs_0) * scales[0];
+ isum += __riscv_vmv_x_s_i32m1_i32(vs_1) * scales[2];
+ isum += __riscv_vmv_x_s_i32m1_i32(vs_2) * scales[1];
+ isum += __riscv_vmv_x_s_i32m1_i32(vs_3) * scales[3];
+
+ sumf += d * isum;
+
+ }
+
+ *s = sumf;
+
+#else
+
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
+ float sums [8];
+ int32_t aux32[8];
+ int32_t scales[4];
+ memset(sums, 0, 8*sizeof(float));
+
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict hm = x[i].hmask;
+ const int8_t * restrict q8 = y[i].qs;
+ int8_t * restrict a = aux8;
+ for (int l = 0; l < 8; ++l) {
+ a[l+ 0] = (int8_t)((q3[l+0] >> 0) & 3) - (hm[l] & 0x01 ? 0 : 4);
+ a[l+ 8] = (int8_t)((q3[l+8] >> 0) & 3) - (hm[l] & 0x02 ? 0 : 4);
+ a[l+16] = (int8_t)((q3[l+0] >> 2) & 3) - (hm[l] & 0x04 ? 0 : 4);
+ a[l+24] = (int8_t)((q3[l+8] >> 2) & 3) - (hm[l] & 0x08 ? 0 : 4);
+ a[l+32] = (int8_t)((q3[l+0] >> 4) & 3) - (hm[l] & 0x10 ? 0 : 4);
+ a[l+40] = (int8_t)((q3[l+8] >> 4) & 3) - (hm[l] & 0x20 ? 0 : 4);
+ a[l+48] = (int8_t)((q3[l+0] >> 6) & 3) - (hm[l] & 0x40 ? 0 : 4);
+ a[l+56] = (int8_t)((q3[l+8] >> 6) & 3) - (hm[l] & 0x80 ? 0 : 4);
+ }
+
+ scales[0] = (x[i].scales[0] & 0xF) - 8;
+ scales[1] = (x[i].scales[0] >> 4) - 8;
+ scales[2] = (x[i].scales[1] & 0xF) - 8;
+ scales[3] = (x[i].scales[1] >> 4) - 8;
+
+ memset(aux32, 0, 8*sizeof(int32_t));
+ for (int j = 0; j < QK_K/16; ++j) {
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] += q8[l] * a[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux32[l] += scales[j] * aux16[l];
+ }
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ }
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
+ *s = sumf;
+
+#endif
+
+}
+#endif
+
+#if QK_K == 256
+void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q4_K * restrict x = vx;
+ const block_q8_K * restrict y = vy;
-// We use this macro instead of a function call because for some reason
-// the code runs 2-3% slower, even if the function is declared inline
-#define MULTIPLY_ACCUM_WITH_SCALE(index)\
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];
+ const int nb = n / QK_K;
-#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\
- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\
- q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\
- q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\
- MULTIPLY_ACCUM_WITH_SCALE((index));
+ static const uint32_t kmask1 = 0x3f3f3f3f;
+ static const uint32_t kmask2 = 0x0f0f0f0f;
+ static const uint32_t kmask3 = 0x03030303;
- for (int j = 0; j < QK_K/128; ++j) {
- const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;
+ uint32_t utmp[4];
- ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
- q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));
- q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));
+#ifdef __ARM_NEON
+ const uint8x16_t m4b = vdupq_n_u8(0xf);
+ const int32x4_t mzero = vdupq_n_s32(0);
- MULTIPLY_ACCUM_WITH_SCALE(0);
+ ggml_int8x16x2_t q4bytes;
+ ggml_int8x16x2_t q8bytes;
- SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);
- SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);
- SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);
+ float sumf = 0;
- is += 8;
+ for (int i = 0; i < nb; ++i) {
+
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+
+ const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));
+
+ memcpy(utmp, x[i].scales, 12);
+
+ uint32x2_t mins8 = { 0 };
+ mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);
+ mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);
+
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[0] &= kmask1;
+
+ const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));
+ const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),
+ vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));
+ sumf -= dmin * vaddvq_s32(prod);
+
+ const uint8_t * scales = (const uint8_t *)utmp;
+
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ int32_t sumi1 = 0;
+ int32_t sumi2 = 0;
+
+ for (int j = 0; j < QK_K/64; ++j) {
+ const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;
+
+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
+ q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));
+ q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));
+
+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
+ sumi1 += vaddvq_s32(p1) * scales[2*j+0];
+
+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
+ q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));
+ q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));
+
+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
+
+ sumi2 += vaddvq_s32(p2) * scales[2*j+1];
}
- sum += d * isum;
+ sumf += d * (sumi1 + sumi2);
+
}
- *s = sum;
+ *s = sumf;
#elif defined __AVX2__
- const __m256i m3 = _mm256_set1_epi8(3);
- const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m256i m4 = _mm256_set1_epi8(0xF);
__m256 acc = _mm256_setzero_ps();
+ __m128 acc_m = _mm_setzero_ps();
- for (int i = 0; i < nb; ++i) {
+ for (int i = 0; i < nb; ++i) {
const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
- const uint8_t * restrict q2 = x[i].qs;
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+
+ const uint8_t * restrict q4 = x[i].qs;
const int8_t * restrict q8 = y[i].qs;
- const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);
- const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);
- const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);
- const __m256i mins = _mm256_cvtepi8_epi16(mins8);
- const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));
+ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);
+ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
+ acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);
- const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);
- const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);
- const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);
- const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};
+ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);
+ const __m256i scales = MM256_SET_M128I(sc128, sc128);
__m256i sumi = _mm256_setzero_si256();
- for (int j = 0; j < QK_K/128; ++j) {
-
- const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;
-
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ for (int j = 0; j < QK_K/64; ++j) {
- const __m256i q2_0 = _mm256_and_si256(q2bits, m3);
- const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);
- const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);
- const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);
+ const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));
+ const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));
- __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);
- __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);
- __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);
- __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);
+ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
+ const __m256i q4l = _mm256_and_si256(q4bits, m4);
+ const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);
- p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);
- p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);
- p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);
- p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);
+ const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);
+ p16l = _mm256_madd_epi16(scale_l, p16l);
- p0 = _mm256_add_epi32(p0, p1);
- p2 = _mm256_add_epi32(p2, p3);
+ const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);
+ p16h = _mm256_madd_epi16(scale_h, p16h);
+ const __m256i sumj = _mm256_add_epi32(p16l, p16h);
- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));
+ sumi = _mm256_add_epi32(sumi, sumj);
}
- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
+ __m256 vd = _mm256_set1_ps(d);
+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);
}
- *s = hsum_float_8(acc);
+ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
+
+ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);
#elif defined __AVX__
- const __m128i m3 = _mm_set1_epi8(0x3);
const __m128i m4 = _mm_set1_epi8(0xF);
const __m128i m2 = _mm_set1_epi8(0x2);
__m256 acc = _mm256_setzero_ps();
+ __m128 acc_m = _mm_setzero_ps();
- for (int i = 0; i < nb; ++i) {
+ for (int i = 0; i < nb; ++i) {
- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
- const uint8_t * restrict q2 = x[i].qs;
+ const uint8_t * restrict q4 = x[i].qs;
const int8_t * restrict q8 = y[i].qs;
- // load mins and scales from block_q2_K.scales[QK_K/16]
- const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);
- const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);
- const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);
- const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);
- const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));
-
- // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2
- const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));
- const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
- // sumf += -dmin * summs in 32bits*8
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);
+ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);
+ const __m128i scales = _mm_cvtepu8_epi16(utmps);
+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));
- const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);
- const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));
- const __m128i scales[2] = { scales_0, scales_1 };
+ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);
+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);
+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);
+ const __m128i prod = _mm_madd_epi16(mins, q8s);
+ acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);
__m128i sumi_0 = _mm_setzero_si128();
__m128i sumi_1 = _mm_setzero_si128();
- for (int j = 0; j < QK_K/128; ++j) {
-
- // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]
- const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
-
- // load 2bits*16*8 from block_q2_K.qs[QK_K/4]
- __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;
- const __m128i q2_0 = _mm_and_si128(q2bits, m3);
- const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
- const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
- const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
- q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;
- const __m128i q2_1 = _mm_and_si128(q2bits, m3);
- const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
- const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
- const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
-
- // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8
- __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);
- __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);
- __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);
- __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);
- __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);
- __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);
- __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);
- __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ for (int j = 0; j < QK_K/64; ++j) {
- // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8
- __m128i shuffle = _mm_set1_epi16(0x0100);
- p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);
- shuffle = _mm_add_epi16(shuffle, m2);
- p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);
- shuffle = _mm_add_epi16(shuffle, m2);
- p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);
- shuffle = _mm_add_epi16(shuffle, m2);
- p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);
- shuffle = _mm_add_epi16(shuffle, m2);
- p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);
- shuffle = _mm_add_epi16(shuffle, m2);
- p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);
+ const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);
shuffle = _mm_add_epi16(shuffle, m2);
- p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);
+ const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);
shuffle = _mm_add_epi16(shuffle, m2);
- p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);
-
- p0 = _mm_add_epi32(p0, p1);
- p2 = _mm_add_epi32(p2, p3);
- p4 = _mm_add_epi32(p4, p5);
- p6 = _mm_add_epi32(p6, p7);
- // isum in 32bits*4*2
- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));
- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));
- }
-
- // sumf += dall * isum - dmin * summs in 32bits
+ __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4l_0 = _mm_and_si128(q4bits, m4);
+ const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);
+ q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4l_1 = _mm_and_si128(q4bits, m4);
+ const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);
+
+ const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);
+ p16l = _mm_madd_epi16(scale_l, p16l);
+ sumi_0 = _mm_add_epi32(sumi_0, p16l);
+ const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ p16l = _mm_maddubs_epi16(q4l_1, q8l_1);
+ p16l = _mm_madd_epi16(scale_l, p16l);
+ sumi_1 = _mm_add_epi32(sumi_1, p16l);
+
+ const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);
+ p16h = _mm_madd_epi16(scale_h, p16h);
+ sumi_0 = _mm_add_epi32(sumi_0, p16h);
+ const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ p16h = _mm_maddubs_epi16(q4h_1, q8h_1);
+ p16h = _mm_madd_epi16(scale_h, p16h);
+ sumi_1 = _mm_add_epi32(sumi_1, p16h);
+
+ }
+
+ __m256 vd = _mm256_set1_ps(d);
__m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);
+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);
+
}
- *s = hsum_float_8(acc);
+ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
+
+ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);
#elif defined __riscv_v_intrinsic
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
+
float sumf = 0;
- uint8_t temp_01[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
for (int i = 0; i < nb; ++i) {
- const uint8_t * q2 = x[i].qs;
- const int8_t * q8 = y[i].qs;
- const uint8_t * sc = x[i].scales;
+ size_t vl = 8;
- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
- size_t vl = 16;
+ vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);
+ vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);
+ vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);
- vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl);
- vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl);
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
- vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl);
+ vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);
+ vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));
+ vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);
- vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl);
- vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl);
- vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));
- vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl);
- vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
+ vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
+ sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);
- sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums);
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
vl = 32;
- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
- vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl);
-
- uint8_t is=0;
- int isum=0;
-
- for (int j = 0; j < QK_K/128; ++j) {
- // load Q2
- vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl);
-
- vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl);
- vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03 , vl);
- vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03 , vl);
- vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03 , vl);
+ int32_t sum_1 = 0;
+ int32_t sum_2 = 0;
- // duplicate scale elements for product
- vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0+is, vl), vl);
- vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2+is, vl), vl);
- vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4+is, vl), vl);
- vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6+is, vl), vl);
+ vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);
- vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl));
- vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl));
- vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl));
- vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl));
+ for (int j = 0; j < QK_K/64; ++j) {
+ // load Q4
+ vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);
- // load Q8
- vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);
- vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);
- vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8+64, vl);
- vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8+96, vl);
+ // load Q8 and multiply it with lower Q4 nibble
+ vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);
+ vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));
+ vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);
+ vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);
- vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl);
- vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl);
- vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl);
- vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl);
+ sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];
- vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl);
- vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl);
+ // load Q8 and multiply it with upper Q4 nibble
+ vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);
+ vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));
+ vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);
+ vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);
- isum += __riscv_vmv_x_s_i32m1_i32(isum1);
+ sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];
- q2+=32; q8+=128; is=8;
+ q4 += 32; q8 += 64;
}
- sumf += dall * isum;
+ sumf += d*(sum_1 + sum_2);
}
@@ -4313,269 +6557,265 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r
#else
- float sumf = 0;
-
- for (int i = 0; i < nb; ++i) {
- const uint8_t * q2 = x[i].qs;
- const int8_t * q8 = y[i].qs;
- const uint8_t * sc = x[i].scales;
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
- int summs = 0;
- for (int j = 0; j < 16; ++j) {
- summs += y[i].bsums[j] * (sc[j] >> 4);
- }
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
+ float sums [8];
+ int32_t aux32[8];
+ memset(sums, 0, 8*sizeof(float));
- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+ float sumf = 0;
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
+ int8_t * restrict a = aux8;
+ for (int j = 0; j < QK_K/64; ++j) {
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
+ a += 32;
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
+ a += 32; q4 += 32;
+ }
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
- int isum = 0;
+ int sumi = 0;
+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
+ a = aux8;
int is = 0;
- int d;
- for (int k = 0; k < QK_K/128; ++k) {
- int shift = 0;
- for (int j = 0; j < 4; ++j) {
- d = sc[is++] & 0xF;
- int isuml = 0;
- for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
- isum += d * isuml;
- d = sc[is++] & 0xF;
- isuml = 0;
- for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
- isum += d * isuml;
- shift += 2;
- q8 += 32;
- }
- q2 += 32;
+ for (int j = 0; j < QK_K/32; ++j) {
+ int32_t scale = scales[is++];
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
}
- sumf += dall * isum - dmin * summs;
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
+ sumf -= dmin * sumi;
}
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
*s = sumf;
#endif
}
-
#else
-
-void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
assert(nrc == 1);
UNUSED(nrc);
UNUSED(bx);
UNUSED(by);
UNUSED(bs);
- const block_q2_K * restrict x = vx;
+ const block_q4_K * restrict x = vx;
const block_q8_K * restrict y = vy;
const int nb = n / QK_K;
#ifdef __ARM_NEON
- const uint8x16_t m3 = vdupq_n_u8(0x3);
+ const uint8x16_t m4b = vdupq_n_u8(0xf);
- const int32x4_t vzero = vdupq_n_s32(0);
+ const int32x4_t mzero = vdupq_n_s32(0);
- ggml_int8x16x4_t q2bytes;
+ float sumf = 0;
- uint32_t aux32[2];
- const uint8_t * scales = (const uint8_t *)aux32;
+ ggml_int8x16x2_t q4bytes;
+ ggml_int8x16x4_t q8bytes;
- float sum = 0;
+ float sum_mins = 0.f;
- for (int i = 0; i < nb; ++i) {
+ uint16_t aux16[2];
+ const uint8_t * restrict scales = (const uint8_t *)aux16;
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+ for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q2 = x[i].qs;
+ const uint8_t * restrict q4 = x[i].qs;
const int8_t * restrict q8 = y[i].qs;
- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
- aux32[0] = sc[0] & 0x0f0f0f0f;
- aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;
+ const uint16_t * restrict a = (const uint16_t *)x[i].scales;
+ aux16[0] = a[0] & 0x0f0f;
+ aux16[1] = (a[0] >> 4) & 0x0f0f;
- sum += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);
+ const int32_t summi = scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]);
+ sum_mins += y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * summi;
- int isum1 = 0, isum2 = 0;
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);
- const uint8x16_t q2bits = vld1q_u8(q2);
+ const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4);
- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);
+ q8bytes = ggml_vld1q_s8_x4(q8);
+ q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));
+ q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));
- q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits, m3));
- q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 2), m3));
- q2bytes.val[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 4), m3));
- q2bytes.val[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 6), m3));
+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
+ const int32_t sumi1 = vaddvq_s32(p1) * scales[0];
- isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * scales[0];
- isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * scales[1];
- isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[2], q8bytes.val[2])) * scales[2];
- isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[3], q8bytes.val[3])) * scales[3];
+ q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));
+ q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));
- sum += d * (isum1 + isum2);
+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[2]), q4bytes.val[1], q8bytes.val[3]);
+ const int32_t sumi2 = vaddvq_s32(p2) * scales[1];
+
+ sumf += d * (sumi1 + sumi2);
}
- *s = sum;
+ *s = sumf - sum_mins;
#elif defined __AVX2__
- const __m256i m3 = _mm256_set1_epi8(3);
+ const __m256i m4 = _mm256_set1_epi8(0xF);
__m256 acc = _mm256_setzero_ps();
- uint32_t ud, um;
- const uint8_t * restrict db = (const uint8_t *)&ud;
- const uint8_t * restrict mb = (const uint8_t *)&um;
-
float summs = 0;
- // TODO: optimize this
+ uint16_t aux16[2];
+ const uint8_t * scales = (const uint8_t *)aux16;
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+ const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;
+ const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;
+ const __m256 vd = _mm256_set1_ps(d);
- const uint8_t * restrict q2 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
+ const uint16_t * a = (const uint16_t *)x[i].scales;
+ aux16[0] = a[0] & 0x0f0f;
+ aux16[1] = (a[0] >> 4) & 0x0f0f;
- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
- ud = (sc[0] >> 0) & 0x0f0f0f0f;
- um = (sc[0] >> 4) & 0x0f0f0f0f;
+ summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
- int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];
- summs += dmin * smin;
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
- const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);
- const __m256i q2_0 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 2), q2bits), m3);
- const __m256i q2_1 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 6), _mm_srli_epi16(q2bits, 4)), m3);
+ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);
+ const __m256i q4l = _mm256_and_si256(q4bits, m4);
+ const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+ const __m256i q8l = _mm256_loadu_si256((const __m256i*)(q8+ 0));
+ const __m256i q8h = _mm256_loadu_si256((const __m256i*)(q8+32));
- const __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);
- const __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);
+ const __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);
+ const __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);
- const __m256i p_0 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 0));
- const __m256i p_1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 1));
- const __m256i p_2 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 0));
- const __m256i p_3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 1));
+ const __m256i p32l = _mm256_madd_epi16(_mm256_set1_epi16(scales[0]), p16l);
+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32l), acc);
+
+ const __m256i p32h = _mm256_madd_epi16(_mm256_set1_epi16(scales[1]), p16h);
+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32h), acc);
- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0), acc);
- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1), acc);
- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2), acc);
- acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3), acc);
}
- *s = hsum_float_8(acc) + summs;
+ *s = hsum_float_8(acc) - summs;
#elif defined __AVX__
- const __m128i m3 = _mm_set1_epi8(3);
+ const __m128i m4 = _mm_set1_epi8(0xF);
__m256 acc = _mm256_setzero_ps();
- uint32_t ud, um;
- const uint8_t * restrict db = (const uint8_t *)&ud;
- const uint8_t * restrict mb = (const uint8_t *)&um;
-
float summs = 0;
- // TODO: optimize this
+ uint16_t aux16[2];
+ const uint8_t * scales = (const uint8_t *)aux16;
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+ const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;
+ const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;
+ const __m256 vd = _mm256_set1_ps(d);
- const uint8_t * restrict q2 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
+ const uint16_t * a = (const uint16_t *)x[i].scales;
+ aux16[0] = a[0] & 0x0f0f;
+ aux16[1] = (a[0] >> 4) & 0x0f0f;
- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
- ud = (sc[0] >> 0) & 0x0f0f0f0f;
- um = (sc[0] >> 4) & 0x0f0f0f0f;
+ summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
- int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];
- summs += dmin * smin;
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
- const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2);
- const __m128i q2_0 = _mm_and_si128(q2bits, m3);
- const __m128i q2_1 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);
- const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);
- const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);
+ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);
+ const __m128i q4bits_0 = _mm256_extractf128_si256(q4bits, 0);
+ const __m128i q4bits_1 = _mm256_extractf128_si256(q4bits, 1);
+ const __m128i q4_0 = _mm_and_si128(q4bits_0, m4);
+ const __m128i q4_1 = _mm_and_si128(q4bits_1, m4);
+ const __m128i q4_2 = _mm_and_si128(_mm_srli_epi16(q4bits_0, 4), m4);
+ const __m128i q4_3 = _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4);
const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
- const __m128i p0 = _mm_maddubs_epi16(q2_0, _mm256_extractf128_si256(q8_0, 0));
- const __m128i p1 = _mm_maddubs_epi16(q2_1, _mm256_extractf128_si256(q8_0, 1));
- const __m128i p2 = _mm_maddubs_epi16(q2_2, _mm256_extractf128_si256(q8_1, 0));
- const __m128i p3 = _mm_maddubs_epi16(q2_3, _mm256_extractf128_si256(q8_1, 1));
+ const __m128i p16_0 = _mm_maddubs_epi16(q4_0, _mm256_extractf128_si256(q8_0, 0));
+ const __m128i p16_1 = _mm_maddubs_epi16(q4_1, _mm256_extractf128_si256(q8_0, 1));
+ const __m128i p16_2 = _mm_maddubs_epi16(q4_2, _mm256_extractf128_si256(q8_1, 0));
+ const __m128i p16_3 = _mm_maddubs_epi16(q4_3, _mm256_extractf128_si256(q8_1, 1));
- const __m256i p_0 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p0, p0)), _mm_cvtepi16_epi32(p0));
- const __m256i p_1 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p1, p1)), _mm_cvtepi16_epi32(p1));
- const __m256i p_2 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p2, p2)), _mm_cvtepi16_epi32(p2));
- const __m256i p_3 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p3, p3)), _mm_cvtepi16_epi32(p3));
+ const __m128i p32_0 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_0);
+ const __m128i p32_1 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_1);
+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_1, p32_0))), acc);
+
+ const __m128i p32_2 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_2);
+ const __m128i p32_3 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_3);
+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_3, p32_2))), acc);
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0)), acc);
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1)), acc);
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2)), acc);
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3)), acc);
}
- *s = hsum_float_8(acc) + summs;
+ *s = hsum_float_8(acc) - summs;
#elif defined __riscv_v_intrinsic
- uint32_t aux32[2];
- const uint8_t * scales = (const uint8_t *)aux32;
+ uint16_t s16[2];
+ const uint8_t * restrict scales = (const uint8_t *)s16;
float sumf = 0;
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- const uint8_t * restrict q2 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
- const uint32_t * restrict sc = (const uint32_t *)x[i].scales;
-
- aux32[0] = sc[0] & 0x0f0f0f0f;
- aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f;
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
- sumf += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]);
+ const uint16_t * restrict b = (const uint16_t *)x[i].scales;
+ s16[0] = b[0] & 0x0f0f;
+ s16[1] = (b[0] >> 4) & 0x0f0f;
- int isum1 = 0;
- int isum2 = 0;
+ sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);
- size_t vl = 16;
+ size_t vl = 32;
vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);
- // load Q2
- vuint8mf2_t q2_x = __riscv_vle8_v_u8mf2(q2, vl);
-
- vint8mf2_t q2_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q2_x, 0x03, vl));
- vint8mf2_t q2_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x2, vl), 0x03 , vl));
- vint8mf2_t q2_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x4, vl), 0x03 , vl));
- vint8mf2_t q2_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x6, vl), 0x03 , vl));
-
- // load Q8, and take product with Q2
- vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q2_0, __riscv_vle8_v_i8mf2(q8, vl), vl);
- vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q2_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);
- vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q2_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);
- vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q2_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);
+ // load Q4
+ vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);
- vint16m1_t vs_0 = __riscv_vredsum_vs_i16m1_i16m1(p0, vzero, vl);
- vint16m1_t vs_1 = __riscv_vredsum_vs_i16m1_i16m1(p1, vzero, vl);
- vint16m1_t vs_2 = __riscv_vredsum_vs_i16m1_i16m1(p2, vzero, vl);
- vint16m1_t vs_3 = __riscv_vredsum_vs_i16m1_i16m1(p3, vzero, vl);
+ // load Q8 and multiply it with lower Q4 nibble
+ vint8m1_t q4_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));
+ vint16m2_t va_0 = __riscv_vwmul_vv_i16m2(q4_a, __riscv_vle8_v_i8m1(q8, vl), vl);
+ vint16m1_t aux1 = __riscv_vredsum_vs_i16m2_i16m1(va_0, vzero, vl);
- isum1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[0];
- isum2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[1];
- isum1 += __riscv_vmv_x_s_i16m1_i16(vs_2) * scales[2];
- isum2 += __riscv_vmv_x_s_i16m1_i16(vs_3) * scales[3];
+ sumf += d*scales[0]*__riscv_vmv_x_s_i16m1_i16(aux1);
- sumf += d * (isum1 + isum2);
+ // load Q8 and multiply it with upper Q4 nibble
+ vint8m1_t q4_s = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));
+ vint16m2_t va_1 = __riscv_vwmul_vv_i16m2(q4_s, __riscv_vle8_v_i8m1(q8+32, vl), vl);
+ vint16m1_t aux2 = __riscv_vredsum_vs_i16m2_i16m1(va_1, vzero, vl);
+
+ sumf += d*scales[1]*__riscv_vmv_x_s_i16m1_i16(aux2);
}
@@ -4583,87 +6823,47 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r
#else
- float sumf = 0;
+ uint8_t aux8[QK_K];
+ int16_t aux16[16];
+ float sums [8];
+ memset(sums, 0, 8*sizeof(float));
- int isum[QK_K/16];
+ uint16_t s16[2];
+ const uint8_t * restrict scales = (const uint8_t *)s16;
+ float sumf = 0;
for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q4 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+ uint8_t * restrict a = aux8;
+ for (int l = 0; l < 32; ++l) a[l+ 0] = q4[l] & 0xF;
+ for (int l = 0; l < 32; ++l) a[l+32] = q4[l] >> 4;
- const uint8_t * q2 = x[i].qs;
- const int8_t * q8 = y[i].qs;
- const uint8_t * sc = x[i].scales;
+ const uint16_t * restrict b = (const uint16_t *)x[i].scales;
+ s16[0] = b[0] & 0x0f0f;
+ s16[1] = (b[0] >> 4) & 0x0f0f;
- int summs = 0;
- for (int j = 0; j < QK_K/16; ++j) {
- summs += y[i].bsums[j] * (sc[j] >> 4);
- }
+ sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);
- memset(isum, 0, (QK_K/16)*sizeof(int));
- for (int l = 0; l < 16; ++l) {
- isum[0] += q8[l+ 0] * ((q2[l] >> 0) & 3);
- isum[1] += q8[l+16] * ((q2[l] >> 2) & 3);
- isum[2] += q8[l+32] * ((q2[l] >> 4) & 3);
- isum[3] += q8[l+48] * ((q2[l] >> 6) & 3);
- }
- for (int l = 0; l < QK_K/16; ++l) {
- isum[l] *= (sc[l] & 0xF);
+ for (int j = 0; j < QK_K/32; ++j) {
+ for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l];
+ q8 += 16; a += 16;
+ for (int l = 0; l < 16; ++l) aux16[l] += q8[l] * a[l];
+ q8 += 16; a += 16;
+ const float dl = d * scales[j];
+ for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[l+8]);
}
- sumf += dall * (isum[0] + isum[1] + isum[2] + isum[3]) - dmin * summs;
}
+ for (int l = 0; l < 8; ++l) sumf += sums[l];
*s = sumf;
#endif
}
#endif
#if QK_K == 256
-
-#ifdef __x86_64__
-#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_default
-#include "ggml_vec_dot_q3_K_q8_K_256.inc"
-#undef ggml_vec_dot_q3_K_q8_K
-#define ggml_vec_dot_q3_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q3_K_q8_K_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "ggml_vec_dot_q3_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef ggml_vec_dot_q3_K_q8_K
-#define ggml_vec_dot_q3_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q3_K_q8_K_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "ggml_vec_dot_q3_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef ggml_vec_dot_q3_K_q8_K
-void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {
- ggml_vec_dot_q3_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);
- } else if (X86_HAVE(AVX)) {
- ggml_vec_dot_q3_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);
- } else {
- ggml_vec_dot_q3_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);
- }
-}
-#else
-#define static
-#include "ggml_vec_dot_q3_K_q8_K_256.inc"
-#undef static
-#endif
-
-#else
-
-void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
assert(n % QK_K == 0);
assert(nrc == 1);
UNUSED(nrc);
@@ -4671,390 +6871,424 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r
UNUSED(by);
UNUSED(bs);
- const block_q3_K * restrict x = vx;
+ const block_q5_K * restrict x = vx;
const block_q8_K * restrict y = vy;
const int nb = n / QK_K;
-#ifdef __ARM_NEON
- const int32x4_t vzero = vdupq_n_s32(0);
+ static const uint32_t kmask1 = 0x3f3f3f3f;
+ static const uint32_t kmask2 = 0x0f0f0f0f;
+ static const uint32_t kmask3 = 0x03030303;
- const uint8x16_t m3b = vdupq_n_u8(0x3);
- const uint8x16_t mh = vdupq_n_u8(4);
+ uint32_t utmp[4];
- ggml_int8x16x4_t q3bytes;
+#ifdef __ARM_NEON
+ const uint8x16_t m4b = vdupq_n_u8(0xf);
+ const uint8x16_t mone = vdupq_n_u8(1);
+ const uint8x16_t mtwo = vdupq_n_u8(2);
+ const int32x4_t mzero = vdupq_n_s32(0);
- uint16_t aux16[2];
- int8_t * scales = (int8_t *)aux16;
+ ggml_int8x16x4_t q5bytes;
- float sum = 0;
+ float sumf = 0;
for (int i = 0; i < nb; ++i) {
- ggml_uint8x16x4_t q3h;
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
- const uint8x8_t hbits = vld1_u8(x[i].hmask);
- const uint8x16_t q3bits = vld1q_u8(x[i].qs);
- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(y[i].qs);
+ const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));
- const uint16_t a = *(const uint16_t *)x[i].scales;
- aux16[0] = a & 0x0f0f;
- aux16[1] = (a >> 4) & 0x0f0f;
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
- for (int j = 0; j < 4; ++j) scales[j] -= 8;
+ const uint8x8_t mins8 = vld1_u8((const uint8_t*)utmp + 8);
+ const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(mins8));
+ const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),
+ vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));
+ int32_t sumi_mins = vaddvq_s32(prod);
- int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);
+ const uint8_t * scales = (const uint8_t *)utmp;
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
- const uint8x16_t htmp = vcombine_u8(hbits, vshr_n_u8(hbits, 1));
- q3h.val[0] = vandq_u8(mh, vshlq_n_u8(htmp, 2));
- q3h.val[1] = vandq_u8(mh, htmp);
- q3h.val[2] = vandq_u8(mh, vshrq_n_u8(htmp, 2));
- q3h.val[3] = vandq_u8(mh, vshrq_n_u8(htmp, 4));
+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);
- q3bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q3bits, m3b), q3h.val[0]));
- q3bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 2), m3b), q3h.val[1]));
- q3bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 4), m3b), q3h.val[2]));
- q3bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q3bits, 6), q3h.val[3]));
+ ggml_uint8x16x4_t q5h;
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes.val[0])) * scales[0];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes.val[1])) * scales[2];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes.val[2])) * scales[1];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes.val[3])) * scales[3];
+ int32_t sumi = 0;
- sum += d * isum;
+ for (int j = 0; j < QK_K/64; ++j) {
+
+ const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;
+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);
+ q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);
+ q5h.val[2] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[0]), 3);
+ q5h.val[3] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[1]), 3);
+ qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 2);
+ qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 2);
+
+ q5bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[0], m4b), q5h.val[0]));
+ q5bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[1], m4b), q5h.val[1]));
+ q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));
+ q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));
+
+ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;
+ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;
+ }
+ sumf += d * sumi - dmin * sumi_mins;
}
- *s = sum;
+ *s = sumf;
#elif defined __AVX2__
- const __m256i m3 = _mm256_set1_epi8(3);
- const __m256i m1 = _mm256_set1_epi8(1);
+ const __m256i m4 = _mm256_set1_epi8(0xF);
+ const __m128i mzero = _mm_setzero_si128();
+ const __m256i mone = _mm256_set1_epi8(1);
__m256 acc = _mm256_setzero_ps();
- uint64_t aux64;
+ float summs = 0.f;
- uint16_t aux16[2];
- const int8_t * aux8 = (const int8_t *)aux16;
+ for (int i = 0; i < nb; ++i) {
- for (int i = 0; i < nb; ++i) {
+ const uint8_t * restrict q5 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+#if QK_K == 256
const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
- const uint8_t * restrict q3 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
+#else
+ // TODO
+ const float d = 0, dmin = 0;
+#endif
- const uint16_t a = *(const uint16_t *)x[i].scales;
- aux16[0] = a & 0x0f0f;
- aux16[1] = (a >> 4) & 0x0f0f;
+ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
- const __m256i scale_0 = MM256_SET_M128I(_mm_set1_epi16(aux8[2] - 8), _mm_set1_epi16(aux8[0] - 8));
- const __m256i scale_1 = MM256_SET_M128I(_mm_set1_epi16(aux8[3] - 8), _mm_set1_epi16(aux8[1] - 8));
+ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);
+ summs += dmin * _mm_extract_epi32(hsum, 0);
- memcpy(&aux64, x[i].hmask, 8);
+ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);
+ const __m256i scales = MM256_SET_M128I(sc128, sc128);
- const __m128i haux = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);
- __m256i q3h_0 = MM256_SET_M128I(_mm_srli_epi16(haux, 2), haux);
- __m256i q3h_1 = _mm256_srli_epi16(q3h_0, 4);
- q3h_0 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_0, m1), 2);
- q3h_1 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_1, m1), 2);
+ const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);
+ __m256i hmask = mone;
- // load low 2 bits
- const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);
+ __m256i sumi = _mm256_setzero_si256();
- // prepare low and high bits
- const __m256i q3aux = MM256_SET_M128I(_mm_srli_epi16(q3bits, 2), q3bits);
- const __m256i q3l_0 = _mm256_and_si256(q3aux, m3);
- const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3aux, 4), m3);
+ int bit = 0;
- // load Q8 quants
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+ for (int j = 0; j < QK_K/64; ++j) {
- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
- // and 2 if the high bit was set)
- const __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);
- const __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);
+ const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));
+ const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));
- __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);
- __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);
+ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;
- p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
- p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
+ const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);
+ const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);
+ const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);
+ hmask = _mm256_slli_epi16(hmask, 1);
- // multiply with scales
- p16_0 = _mm256_madd_epi16(scale_0, p16_0);
- p16_1 = _mm256_madd_epi16(scale_1, p16_1);
+ const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);
+ const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);
+ const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);
+ hmask = _mm256_slli_epi16(hmask, 1);
- p16_0 = _mm256_add_epi32(p16_0, p16_1);
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- // multiply with block scale and accumulate
- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16_0), acc);
+ __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);
+ __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);
+
+ p16_0 = _mm256_madd_epi16(scale_0, p16_0);
+ p16_1 = _mm256_madd_epi16(scale_1, p16_1);
+
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));
+
+ }
+
+ __m256 vd = _mm256_set1_ps(d);
+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);
}
- *s = hsum_float_8(acc);
+ *s = hsum_float_8(acc) + summs;
#elif defined __AVX__
- const __m128i m3 = _mm_set1_epi8(3);
- const __m128i m1 = _mm_set1_epi8(1);
+ const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m128i mzero = _mm_setzero_si128();
+ const __m128i mone = _mm_set1_epi8(1);
+ const __m128i m2 = _mm_set1_epi8(2);
__m256 acc = _mm256_setzero_ps();
- uint64_t aux64;
-
- uint16_t aux16[2];
- const int8_t * aux8 = (const int8_t *)aux16;
+ float summs = 0.f;
for (int i = 0; i < nb; ++i) {
const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
- const uint8_t * restrict q3 = x[i].qs;
+ const uint8_t * restrict q5 = x[i].qs;
const int8_t * restrict q8 = y[i].qs;
- const uint16_t a = *(const uint16_t *)x[i].scales;
- aux16[0] = a & 0x0f0f;
- aux16[1] = (a >> 4) & 0x0f0f;
-
- const __m128i scale_0 = _mm_set1_epi16(aux8[0] - 8);
- const __m128i scale_1 = _mm_set1_epi16(aux8[2] - 8);
- const __m128i scale_2 = _mm_set1_epi16(aux8[1] - 8);
- const __m128i scale_3 = _mm_set1_epi16(aux8[3] - 8);
-
- memcpy(&aux64, x[i].hmask, 8);
-
- __m128i q3h_0 = _mm_set_epi64x(aux64 >> 1, aux64 >> 0);
- __m128i q3h_1 = _mm_srli_epi16(q3h_0, 2);
- __m128i q3h_2 = _mm_srli_epi16(q3h_0, 4);
- __m128i q3h_3 = _mm_srli_epi16(q3h_0, 6);
- q3h_0 = _mm_slli_epi16(_mm_andnot_si128(q3h_0, m1), 2);
- q3h_1 = _mm_slli_epi16(_mm_andnot_si128(q3h_1, m1), 2);
- q3h_2 = _mm_slli_epi16(_mm_andnot_si128(q3h_2, m1), 2);
- q3h_3 = _mm_slli_epi16(_mm_andnot_si128(q3h_3, m1), 2);
-
- // load low 2 bits
- const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3);
-
- // prepare low and high bits
- const __m128i q3l_0 = _mm_and_si128(q3bits, m3);
- const __m128i q3l_1 = _mm_and_si128(_mm_srli_epi16(q3bits, 2), m3);
- const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits, 4), m3);
- const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits, 6), m3);
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
- // load Q8 quants
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);
+ const __m128i scales = _mm_cvtepu8_epi16(utmps);
+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));
- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm_maddubs_epi16,
- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
- // and 2 if the high bit was set)
- const __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, _mm256_extractf128_si256(q8_0, 0));
- const __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, _mm256_extractf128_si256(q8_0, 1));
- const __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, _mm256_extractf128_si256(q8_1, 0));
- const __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, _mm256_extractf128_si256(q8_1, 1));
+ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);
+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);
+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);
+ const __m128i prod = _mm_madd_epi16(mins, q8s);
+ const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);
+ summs += dmin * _mm_extract_epi32(hsum, 0);
- __m128i p16_0 = _mm_maddubs_epi16(q3l_0, _mm256_extractf128_si256(q8_0, 0));
- __m128i p16_1 = _mm_maddubs_epi16(q3l_1, _mm256_extractf128_si256(q8_0, 1));
- __m128i p16_2 = _mm_maddubs_epi16(q3l_2, _mm256_extractf128_si256(q8_1, 0));
- __m128i p16_3 = _mm_maddubs_epi16(q3l_3, _mm256_extractf128_si256(q8_1, 1));
+ const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);
+ const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);
+ __m128i hmask = mone;
- p16_0 = _mm_sub_epi16(p16_0, q8s_0);
- p16_1 = _mm_sub_epi16(p16_1, q8s_1);
- p16_2 = _mm_sub_epi16(p16_2, q8s_2);
- p16_3 = _mm_sub_epi16(p16_3, q8s_3);
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
- // multiply with scales
- p16_0 = _mm_madd_epi16(scale_0, p16_0);
- p16_1 = _mm_madd_epi16(scale_1, p16_1);
- p16_2 = _mm_madd_epi16(scale_2, p16_2);
- p16_3 = _mm_madd_epi16(scale_3, p16_3);
+ int bit = 0;
- p16_0 = _mm_add_epi32(p16_0, p16_2);
- p16_1 = _mm_add_epi32(p16_1, p16_3);
- __m256i p16 = MM256_SET_M128I(p16_1, p16_0);
+ __m128i shuffle = _mm_set1_epi16(0x0100);
+ for (int j = 0; j < QK_K/64; ++j) {
- // multiply with block scale and accumulate
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16)), acc);
+ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi16(shuffle, m2);
+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi16(shuffle, m2);
+
+ const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;
+ const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;
+
+ __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);
+ __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);
+ __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);
+ __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);
+ __m128i q5_0 = _mm_add_epi8(q5l_0, q5h_0);
+ __m128i q5_1 = _mm_add_epi8(q5l_1, q5h_1);
+ hmask = _mm_slli_epi16(hmask, 1);
+
+ __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);
+ __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);
+ p16_0 = _mm_madd_epi16(scale_0, p16_0);
+ p16_1 = _mm_madd_epi16(scale_0, p16_1);
+
+ q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);
+ q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);
+ q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);
+ q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);
+ q5_0 = _mm_add_epi8(q5l_0, q5h_0);
+ q5_1 = _mm_add_epi8(q5l_1, q5h_1);
+ hmask = _mm_slli_epi16(hmask, 1);
+
+ q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);
+ __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);
+ p16_2 = _mm_madd_epi16(scale_1, p16_2);
+ p16_3 = _mm_madd_epi16(scale_1, p16_3);
+
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));
+
+ }
+
+ __m256 vd = _mm256_set1_ps(d);
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);
}
- *s = hsum_float_8(acc);
+ *s = hsum_float_8(acc) + summs;
#elif defined __riscv_v_intrinsic
- uint16_t aux16[2];
- int8_t * scales = (int8_t *)aux16;
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
float sumf = 0;
+ float sums = 0.0;
+
+ size_t vl;
for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q3 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
+ vl = 8;
- const uint16_t a = *(const uint16_t *)x[i].scales;
- aux16[0] = a & 0x0f0f;
- aux16[1] = (a >> 4) & 0x0f0f;
+ const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict hm = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
- for (int j = 0; j < 4; ++j) scales[j] -= 8;
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
- int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]);
+ vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);
+ vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);
+ vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+ vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);
+ vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));
+ vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);
- // load qh
- vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(x[i].hmask, 8);
- vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));
+ vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
+ sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);
- size_t vl = 16;
+ vl = 32;
+ int32_t aux32 = 0;
+ int is = 0;
- // extend and combine both qh_x1 and qh_x2
- vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);
+ uint8_t m = 1;
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
+ vuint8m1_t vqh = __riscv_vle8_v_u8m1(hm, vl);
- vuint8mf2_t qh_0 = __riscv_vand_vx_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);
- vuint8mf2_t qh_1 = __riscv_vand_vx_u8mf2(qh_x, 0x4, vl);
- vuint8mf2_t qh_2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl);
- vuint8mf2_t qh_3 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), 0x4, vl);
+ for (int j = 0; j < QK_K/64; ++j) {
+ // load Q5 and Q8
+ vuint8m1_t q5_x = __riscv_vle8_v_u8m1(q5, vl);
+ vint8m1_t q8_y1 = __riscv_vle8_v_i8m1(q8, vl);
+ vint8m1_t q8_y2 = __riscv_vle8_v_i8m1(q8+32, vl);
- // load Q3
- vuint8mf2_t q3_x = __riscv_vle8_v_u8mf2(q3, vl);
+ // compute mask for addition
+ vint8m1_t q5_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q5_x, 0x0F, vl));
+ vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_1 = __riscv_vmsne_vx_u8m1_b8(qh_m1, 0, vl);
+ vint8m1_t q5_m1 = __riscv_vadd_vx_i8m1_m(vmask_1, q5_a, 16, vl);
+ m <<= 1;
- vuint8mf2_t q3h_0 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(q3_x, 0x3, vl), qh_0, vl);
- vuint8mf2_t q3h_1 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 2, vl), 0x3, vl), qh_1, vl);
- vuint8mf2_t q3h_2 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 4, vl), 0x3, vl), qh_2, vl);
- vuint8mf2_t q3h_3 = __riscv_vor_vv_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x6, vl), qh_3, vl);
+ vint8m1_t q5_l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q5_x, 0x04, vl));
+ vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);
+ vbool8_t vmask_2 = __riscv_vmsne_vx_u8m1_b8(qh_m2, 0, vl);
+ vint8m1_t q5_m2 = __riscv_vadd_vx_i8m1_m(vmask_2, q5_l, 16, vl);
+ m <<= 1;
- vint8mf2_t q3_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_0);
- vint8mf2_t q3_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_1);
- vint8mf2_t q3_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_2);
- vint8mf2_t q3_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_3);
+ vint16m2_t v0 = __riscv_vwmul_vv_i16m2(q5_m1, q8_y1, vl);
+ vint16m2_t v1 = __riscv_vwmul_vv_i16m2(q5_m2, q8_y2, vl);
- // load Q8 and take product with Q3
- vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q3_0, __riscv_vle8_v_i8mf2(q8, vl), vl);
- vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q3_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);
- vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q3_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);
- vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q3_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);
+ vint32m4_t vs1 = __riscv_vwmul_vx_i32m4(v0, scales[is++], vl);
+ vint32m4_t vs2 = __riscv_vwmul_vx_i32m4(v1, scales[is++], vl);
- vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);
- vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);
- vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);
- vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);
+ vint32m1_t vacc1 = __riscv_vredsum_vs_i32m4_i32m1(vs1, vzero, vl);
+ vint32m1_t vacc2 = __riscv_vredsum_vs_i32m4_i32m1(vs2, vzero, vl);
- isum += __riscv_vmv_x_s_i32m1_i32(vs_0) * scales[0];
- isum += __riscv_vmv_x_s_i32m1_i32(vs_1) * scales[2];
- isum += __riscv_vmv_x_s_i32m1_i32(vs_2) * scales[1];
- isum += __riscv_vmv_x_s_i32m1_i32(vs_3) * scales[3];
+ aux32 += __riscv_vmv_x_s_i32m1_i32(vacc1) + __riscv_vmv_x_s_i32m1_i32(vacc2);
+ q5 += 32; q8 += 64;
- sumf += d * isum;
+ }
+
+ vfloat32m1_t vaux = __riscv_vfmul_vf_f32m1(__riscv_vfmv_v_f_f32m1(aux32, 1), d, 1);
+ sums += __riscv_vfmv_f_s_f32m1_f32(vaux);
}
- *s = sumf;
+ *s = sumf+sums;
#else
+ const uint8_t * scales = (const uint8_t*)&utmp[0];
+ const uint8_t * mins = (const uint8_t*)&utmp[2];
+
int8_t aux8[QK_K];
int16_t aux16[8];
float sums [8];
int32_t aux32[8];
- int32_t scales[4];
memset(sums, 0, 8*sizeof(float));
float sumf = 0;
for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q3 = x[i].qs;
- const uint8_t * restrict hm = x[i].hmask;
+ const uint8_t * restrict q4 = x[i].qs;
+ const uint8_t * restrict hm = x[i].qh;
const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
int8_t * restrict a = aux8;
- for (int l = 0; l < 8; ++l) {
- a[l+ 0] = (int8_t)((q3[l+0] >> 0) & 3) - (hm[l] & 0x01 ? 0 : 4);
- a[l+ 8] = (int8_t)((q3[l+8] >> 0) & 3) - (hm[l] & 0x02 ? 0 : 4);
- a[l+16] = (int8_t)((q3[l+0] >> 2) & 3) - (hm[l] & 0x04 ? 0 : 4);
- a[l+24] = (int8_t)((q3[l+8] >> 2) & 3) - (hm[l] & 0x08 ? 0 : 4);
- a[l+32] = (int8_t)((q3[l+0] >> 4) & 3) - (hm[l] & 0x10 ? 0 : 4);
- a[l+40] = (int8_t)((q3[l+8] >> 4) & 3) - (hm[l] & 0x20 ? 0 : 4);
- a[l+48] = (int8_t)((q3[l+0] >> 6) & 3) - (hm[l] & 0x40 ? 0 : 4);
- a[l+56] = (int8_t)((q3[l+8] >> 6) & 3) - (hm[l] & 0x80 ? 0 : 4);
- }
-
- scales[0] = (x[i].scales[0] & 0xF) - 8;
- scales[1] = (x[i].scales[0] >> 4) - 8;
- scales[2] = (x[i].scales[1] & 0xF) - 8;
- scales[3] = (x[i].scales[1] >> 4) - 8;
+ uint8_t m = 1;
+ for (int j = 0; j < QK_K/64; ++j) {
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
+ a += 32; m <<= 1;
+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
+ a += 32; m <<= 1;
+ q4 += 32;
+ }
+ memcpy(utmp, x[i].scales, 12);
+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
+ const uint32_t uaux = utmp[1] & kmask1;
+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
+ utmp[2] = uaux;
+ utmp[0] &= kmask1;
- memset(aux32, 0, 8*sizeof(int32_t));
- for (int j = 0; j < QK_K/16; ++j) {
+ int sumi = 0;
+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
+ a = aux8;
+ int is = 0;
+ for (int j = 0; j < QK_K/32; ++j) {
+ int32_t scale = scales[is++];
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] += q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux32[l] += scales[j] * aux16[l];
}
const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
+ sumf -= dmin * sumi;
}
for (int l = 0; l < 8; ++l) sumf += sums[l];
*s = sumf;
-
#endif
-
}
-#endif
-
-#if QK_K == 256
-#ifdef __x86_64__
-#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_default
-#include "ggml_vec_dot_q4_K_q8_K_256.inc"
-#undef ggml_vec_dot_q4_K_q8_K
-#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q4_K_q8_K_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "ggml_vec_dot_q4_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef ggml_vec_dot_q4_K_q8_K
-#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_K_q8_K_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "ggml_vec_dot_q4_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef ggml_vec_dot_q4_K_q8_K
-void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {
- ggml_vec_dot_q4_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);
- } else if (X86_HAVE(AVX)) {
- ggml_vec_dot_q4_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);
- } else {
- ggml_vec_dot_q4_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);
- }
-}
#else
-#define static
-#include "ggml_vec_dot_q4_K_q8_K_256.inc"
-#undef static
-#endif
-#else
-void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
assert(n % QK_K == 0);
assert(nrc == 1);
UNUSED(nrc);
@@ -5062,199 +7296,224 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r
UNUSED(by);
UNUSED(bs);
- const block_q4_K * restrict x = vx;
+ const block_q5_K * restrict x = vx;
const block_q8_K * restrict y = vy;
const int nb = n / QK_K;
#ifdef __ARM_NEON
const uint8x16_t m4b = vdupq_n_u8(0xf);
-
+ const uint8x16_t mh = vdupq_n_u8(16);
const int32x4_t mzero = vdupq_n_s32(0);
- float sumf = 0;
-
- ggml_int8x16x2_t q4bytes;
- ggml_int8x16x4_t q8bytes;
-
- float sum_mins = 0.f;
+ ggml_int8x16x4_t q5bytes;
+ ggml_uint8x16x4_t q5h;
- uint16_t aux16[2];
- const uint8_t * restrict scales = (const uint8_t *)aux16;
+ float sumf = 0;
for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- const uint16_t * restrict a = (const uint16_t *)x[i].scales;
- aux16[0] = a[0] & 0x0f0f;
- aux16[1] = (a[0] >> 4) & 0x0f0f;
-
- const int32_t summi = scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]);
- sum_mins += y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * summi;
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const int8_t * sc = x[i].scales;
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);
+ const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
- const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4);
+ const uint8x8_t qhbits = vld1_u8(qh);
- q8bytes = ggml_vld1q_s8_x4(q8);
- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));
- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));
+ const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5);
+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);
- const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
- const int32_t sumi1 = vaddvq_s32(p1) * scales[0];
+ const uint8x16_t htmp = vcombine_u8(qhbits, vshr_n_u8(qhbits, 1));
+ q5h.val[0] = vbicq_u8(mh, vshlq_n_u8(htmp, 4));
+ q5h.val[1] = vbicq_u8(mh, vshlq_n_u8(htmp, 2));
+ q5h.val[2] = vbicq_u8(mh, htmp);
+ q5h.val[3] = vbicq_u8(mh, vshrq_n_u8(htmp, 2));
- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));
- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));
+ q5bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[0], m4b)), vreinterpretq_s8_u8(q5h.val[0]));
+ q5bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[1], m4b)), vreinterpretq_s8_u8(q5h.val[1]));
+ q5bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[0], 4)), vreinterpretq_s8_u8(q5h.val[2]));
+ q5bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[1], 4)), vreinterpretq_s8_u8(q5h.val[3]));
- const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[2]), q4bytes.val[1], q8bytes.val[3]);
- const int32_t sumi2 = vaddvq_s32(p2) * scales[1];
+ int32_t sumi1 = sc[0] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]));
+ int32_t sumi2 = sc[1] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[1], q8bytes.val[1]));
+ int32_t sumi3 = sc[2] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]));
+ int32_t sumi4 = sc[3] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[3], q8bytes.val[3]));
- sumf += d * (sumi1 + sumi2);
+ sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);
}
- *s = sumf - sum_mins;
+ *s = sumf;
#elif defined __AVX2__
const __m256i m4 = _mm256_set1_epi8(0xF);
+ const __m256i mone = _mm256_set1_epi8(1);
__m256 acc = _mm256_setzero_ps();
- float summs = 0;
+ for (int i = 0; i < nb; ++i) {
- uint16_t aux16[2];
- const uint8_t * scales = (const uint8_t *)aux16;
+ const uint8_t * restrict q5 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
- for (int i = 0; i < nb; ++i) {
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;
- const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;
- const __m256 vd = _mm256_set1_ps(d);
+ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);
- const uint16_t * a = (const uint16_t *)x[i].scales;
- aux16[0] = a[0] & 0x0f0f;
- aux16[1] = (a[0] >> 4) & 0x0f0f;
+ const __m256i scale_l = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[1]), _mm_set1_epi16(x[i].scales[0]));
+ const __m256i scale_h = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[3]), _mm_set1_epi16(x[i].scales[2]));
- summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
+ int64_t aux64;
+ memcpy(&aux64, x[i].qh, 8);
+ const __m128i haux128 = _mm_set_epi64x(aux64 >> 1, aux64);
+ const __m256i haux256 = MM256_SET_M128I(_mm_srli_epi16(haux128, 2), haux128);
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
+ const __m256i q5h_0 = _mm256_slli_epi16(_mm256_andnot_si256(haux256, mone), 4);
+ const __m256i q5h_1 = _mm256_slli_epi16(_mm256_andnot_si256(_mm256_srli_epi16(haux256, 4), mone), 4);
- const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);
- const __m256i q4l = _mm256_and_si256(q4bits, m4);
- const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);
+ const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);
+ const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);
- const __m256i q8l = _mm256_loadu_si256((const __m256i*)(q8+ 0));
- const __m256i q8h = _mm256_loadu_si256((const __m256i*)(q8+32));
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
- const __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);
- const __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);
+ const __m256i p16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5l_0, q8_0));
+ const __m256i p16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5l_1, q8_1));
+ const __m256i s16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5h_0, q8_0));
+ const __m256i s16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5h_1, q8_1));
- const __m256i p32l = _mm256_madd_epi16(_mm256_set1_epi16(scales[0]), p16l);
- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32l), acc);
+ const __m256i dot = _mm256_sub_epi32(_mm256_add_epi32(p16_0, p16_1), _mm256_add_epi32(s16_0, s16_1));
- const __m256i p32h = _mm256_madd_epi16(_mm256_set1_epi16(scales[1]), p16h);
- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32h), acc);
+ acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(dot), acc);
}
- *s = hsum_float_8(acc) - summs;
+ *s = hsum_float_8(acc);
#elif defined __AVX__
const __m128i m4 = _mm_set1_epi8(0xF);
+ const __m128i mone = _mm_set1_epi8(1);
__m256 acc = _mm256_setzero_ps();
- float summs = 0;
+ for (int i = 0; i < nb; ++i) {
- uint16_t aux16[2];
- const uint8_t * scales = (const uint8_t *)aux16;
+ const uint8_t * restrict q5 = x[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
- for (int i = 0; i < nb; ++i) {
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;
- const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;
- const __m256 vd = _mm256_set1_ps(d);
+ const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);
- const uint16_t * a = (const uint16_t *)x[i].scales;
- aux16[0] = a[0] & 0x0f0f;
- aux16[1] = (a[0] >> 4) & 0x0f0f;
+ const __m128i scale_0 = _mm_set1_epi16(x[i].scales[0]);
+ const __m128i scale_1 = _mm_set1_epi16(x[i].scales[1]);
+ const __m128i scale_2 = _mm_set1_epi16(x[i].scales[2]);
+ const __m128i scale_3 = _mm_set1_epi16(x[i].scales[3]);
- summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
+ int64_t aux64;
+ memcpy(&aux64, x[i].qh, 8);
+ const __m128i haux128_0 = _mm_set_epi64x(aux64 >> 1, aux64);
+ const __m128i haux128_1 = _mm_srli_epi16(haux128_0, 2);
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
+ const __m128i q5h_0 = _mm_slli_epi16(_mm_andnot_si128(haux128_0, mone), 4);
+ const __m128i q5h_1 = _mm_slli_epi16(_mm_andnot_si128(haux128_1, mone), 4);
+ const __m128i q5h_2 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_0, 4), mone), 4);
+ const __m128i q5h_3 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_1, 4), mone), 4);
- const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4);
- const __m128i q4bits_0 = _mm256_extractf128_si256(q4bits, 0);
- const __m128i q4bits_1 = _mm256_extractf128_si256(q4bits, 1);
- const __m128i q4_0 = _mm_and_si128(q4bits_0, m4);
- const __m128i q4_1 = _mm_and_si128(q4bits_1, m4);
- const __m128i q4_2 = _mm_and_si128(_mm_srli_epi16(q4bits_0, 4), m4);
- const __m128i q4_3 = _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4);
+ const __m128i q5l_0 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 0), m4);
+ const __m128i q5l_1 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 1), m4);
+ const __m128i q5l_2 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 0), 4), m4);
+ const __m128i q5l_3 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 1), 4), m4);
const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
- const __m128i p16_0 = _mm_maddubs_epi16(q4_0, _mm256_extractf128_si256(q8_0, 0));
- const __m128i p16_1 = _mm_maddubs_epi16(q4_1, _mm256_extractf128_si256(q8_0, 1));
- const __m128i p16_2 = _mm_maddubs_epi16(q4_2, _mm256_extractf128_si256(q8_1, 0));
- const __m128i p16_3 = _mm_maddubs_epi16(q4_3, _mm256_extractf128_si256(q8_1, 1));
+ const __m128i p16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5l_0, _mm256_extractf128_si256(q8_0, 0)));
+ const __m128i p16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5l_1, _mm256_extractf128_si256(q8_0, 1)));
+ const __m128i p16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5l_2, _mm256_extractf128_si256(q8_1, 0)));
+ const __m128i p16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5l_3, _mm256_extractf128_si256(q8_1, 1)));
+ const __m128i s16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5h_0, _mm256_extractf128_si256(q8_0, 0)));
+ const __m128i s16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5h_1, _mm256_extractf128_si256(q8_0, 1)));
+ const __m128i s16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5h_2, _mm256_extractf128_si256(q8_1, 0)));
+ const __m128i s16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5h_3, _mm256_extractf128_si256(q8_1, 1)));
- const __m128i p32_0 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_0);
- const __m128i p32_1 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_1);
- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_1, p32_0))), acc);
+ const __m128i dot_0 = _mm_sub_epi32(_mm_add_epi32(p16_0, p16_2), _mm_add_epi32(s16_0, s16_2));
+ const __m128i dot_1 = _mm_sub_epi32(_mm_add_epi32(p16_1, p16_3), _mm_add_epi32(s16_1, s16_3));
- const __m128i p32_2 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_2);
- const __m128i p32_3 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_3);
- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_3, p32_2))), acc);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(dot_1, dot_0))), acc);
}
- *s = hsum_float_8(acc) - summs;
+ *s = hsum_float_8(acc);
#elif defined __riscv_v_intrinsic
- uint16_t s16[2];
- const uint8_t * restrict scales = (const uint8_t *)s16;
-
float sumf = 0;
for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const int8_t * sc = x[i].scales;
- const uint16_t * restrict b = (const uint16_t *)x[i].scales;
- s16[0] = b[0] & 0x0f0f;
- s16[1] = (b[0] >> 4) & 0x0f0f;
+ const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
- sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);
+ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
- size_t vl = 32;
+ // load qh
+ vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(qh, 8);
+ vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));
- vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);
+ size_t vl = 16;
- // load Q4
- vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);
+ // combine both qh_1 and qh_2
+ vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);
- // load Q8 and multiply it with lower Q4 nibble
- vint8m1_t q4_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));
- vint16m2_t va_0 = __riscv_vwmul_vv_i16m2(q4_a, __riscv_vle8_v_i8m1(q8, vl), vl);
- vint16m1_t aux1 = __riscv_vredsum_vs_i16m2_i16m1(va_0, vzero, vl);
+ vuint8mf2_t qh_h0 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);
+ vuint8mf2_t qh_h1 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), vl), 16, vl);
+ vuint8mf2_t qh_h2 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(qh_x, vl), 16, vl);
+ vuint8mf2_t qh_h3 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);
- sumf += d*scales[0]*__riscv_vmv_x_s_i16m1_i16(aux1);
+ vint8mf2_t qh_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h0);
+ vint8mf2_t qh_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h1);
+ vint8mf2_t qh_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h2);
+ vint8mf2_t qh_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h3);
- // load Q8 and multiply it with upper Q4 nibble
- vint8m1_t q4_s = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));
- vint16m2_t va_1 = __riscv_vwmul_vv_i16m2(q4_s, __riscv_vle8_v_i8m1(q8+32, vl), vl);
- vint16m1_t aux2 = __riscv_vredsum_vs_i16m2_i16m1(va_1, vzero, vl);
+ // load q5
+ vuint8mf2_t q5_x1 = __riscv_vle8_v_u8mf2(q5, vl);
+ vuint8mf2_t q5_x2 = __riscv_vle8_v_u8mf2(q5+16, vl);
- sumf += d*scales[1]*__riscv_vmv_x_s_i16m1_i16(aux2);
+ vint8mf2_t q5s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x1, 0xF, vl));
+ vint8mf2_t q5s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x2, 0xF, vl));
+ vint8mf2_t q5s_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x1, 0x4, vl));
+ vint8mf2_t q5s_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x2, 0x4, vl));
+
+ vint8mf2_t q5_0 = __riscv_vsub_vv_i8mf2(q5s_0, qh_0, vl);
+ vint8mf2_t q5_1 = __riscv_vsub_vv_i8mf2(q5s_1, qh_1, vl);
+ vint8mf2_t q5_2 = __riscv_vsub_vv_i8mf2(q5s_2, qh_2, vl);
+ vint8mf2_t q5_3 = __riscv_vsub_vv_i8mf2(q5s_3, qh_3, vl);
+
+ // load Q8 and multiply it with Q5
+ vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q5_0, __riscv_vle8_v_i8mf2(q8, vl), vl);
+ vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q5_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);
+ vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q5_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);
+ vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q5_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);
+
+ vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);
+ vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);
+ vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);
+ vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);
+
+ int32_t sumi1 = sc[0] * __riscv_vmv_x_s_i32m1_i32(vs_0);
+ int32_t sumi2 = sc[1] * __riscv_vmv_x_s_i32m1_i32(vs_1);
+ int32_t sumi3 = sc[2] * __riscv_vmv_x_s_i32m1_i32(vs_2);
+ int32_t sumi4 = sc[3] * __riscv_vmv_x_s_i32m1_i32(vs_3);
+
+ sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);
}
@@ -5262,37 +7521,34 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r
#else
- uint8_t aux8[QK_K];
+ int8_t aux8[QK_K];
int16_t aux16[16];
float sums [8];
memset(sums, 0, 8*sizeof(float));
- uint16_t s16[2];
- const uint8_t * restrict scales = (const uint8_t *)s16;
-
float sumf = 0;
for (int i = 0; i < nb; ++i) {
const uint8_t * restrict q4 = x[i].qs;
+ const uint8_t * restrict hm = x[i].qh;
const int8_t * restrict q8 = y[i].qs;
- uint8_t * restrict a = aux8;
- for (int l = 0; l < 32; ++l) a[l+ 0] = q4[l] & 0xF;
- for (int l = 0; l < 32; ++l) a[l+32] = q4[l] >> 4;
-
- const uint16_t * restrict b = (const uint16_t *)x[i].scales;
- s16[0] = b[0] & 0x0f0f;
- s16[1] = (b[0] >> 4) & 0x0f0f;
-
- sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));
+ int8_t * restrict a = aux8;
+ for (int l = 0; l < 32; ++l) {
+ a[l+ 0] = q4[l] & 0xF;
+ a[l+32] = q4[l] >> 4;
+ }
+ for (int is = 0; is < 8; ++is) {
+ uint8_t m = 1 << is;
+ for (int l = 0; l < 8; ++l) a[8*is + l] -= (hm[l] & m ? 0 : 16);
+ }
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const int8_t * restrict sc = x[i].scales;
- for (int j = 0; j < QK_K/32; ++j) {
+ for (int j = 0; j < QK_K/16; ++j) {
+ const float dl = d * sc[j];
for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[8+l]);
q8 += 16; a += 16;
- for (int l = 0; l < 16; ++l) aux16[l] += q8[l] * a[l];
- q8 += 16; a += 16;
- const float dl = d * scales[j];
- for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[l+8]);
}
}
for (int l = 0; l < 8; ++l) sumf += sums[l];
@@ -5301,52 +7557,9 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r
}
#endif
-#if QK_K == 256
-
-#ifdef __x86_64__
-#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_default
-#include "ggml_vec_dot_q5_K_q8_K_256.inc"
-#undef ggml_vec_dot_q5_K_q8_K
-#define ggml_vec_dot_q5_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q5_K_q8_K_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "ggml_vec_dot_q5_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef ggml_vec_dot_q5_K_q8_K
-#define ggml_vec_dot_q5_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q5_K_q8_K_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "ggml_vec_dot_q5_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef ggml_vec_dot_q5_K_q8_K
-void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {
- ggml_vec_dot_q5_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);
- } else if (X86_HAVE(AVX)) {
- ggml_vec_dot_q5_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);
- } else {
- ggml_vec_dot_q5_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);
- }
-}
-#else
-#define static
-#include "ggml_vec_dot_q5_K_q8_K_256.inc"
-#undef static
-#endif
-
-#else
-void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
+#if QK_K == 256
+void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
assert(n % QK_K == 0);
assert(nrc == 1);
UNUSED(nrc);
@@ -5354,98 +7567,181 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r
UNUSED(by);
UNUSED(bs);
- const block_q5_K * restrict x = vx;
+ const block_q6_K * restrict x = vx;
const block_q8_K * restrict y = vy;
const int nb = n / QK_K;
#ifdef __ARM_NEON
- const uint8x16_t m4b = vdupq_n_u8(0xf);
- const uint8x16_t mh = vdupq_n_u8(16);
- const int32x4_t mzero = vdupq_n_s32(0);
+ float sum = 0;
- ggml_int8x16x4_t q5bytes;
- ggml_uint8x16x4_t q5h;
+ const uint8x16_t m4b = vdupq_n_u8(0xF);
+ const int32x4_t vzero = vdupq_n_s32(0);
+ //const int8x16_t m32s = vdupq_n_s8(32);
- float sumf = 0;
+ const uint8x16_t mone = vdupq_n_u8(3);
+
+ ggml_int8x16x4_t q6bytes;
+ ggml_uint8x16x4_t q6h;
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const int8_t * sc = x[i].scales;
+ const float d_all = GGML_FP16_TO_FP32(x[i].d);
- const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict q6 = x[i].ql;
const uint8_t * restrict qh = x[i].qh;
const int8_t * restrict q8 = y[i].qs;
- const uint8x8_t qhbits = vld1_u8(qh);
+ const int8_t * restrict scale = x[i].scales;
- const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5);
- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);
+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);
+ const int8x16_t scales = vld1q_s8(scale);
+ const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};
- const uint8x16_t htmp = vcombine_u8(qhbits, vshr_n_u8(qhbits, 1));
- q5h.val[0] = vbicq_u8(mh, vshlq_n_u8(htmp, 4));
- q5h.val[1] = vbicq_u8(mh, vshlq_n_u8(htmp, 2));
- q5h.val[2] = vbicq_u8(mh, htmp);
- q5h.val[3] = vbicq_u8(mh, vshrq_n_u8(htmp, 2));
+ const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),
+ vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),
+ vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),
+ vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));
+ int32_t isum_mins = vaddvq_s32(prod);
- q5bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[0], m4b)), vreinterpretq_s8_u8(q5h.val[0]));
- q5bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[1], m4b)), vreinterpretq_s8_u8(q5h.val[1]));
- q5bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[0], 4)), vreinterpretq_s8_u8(q5h.val[2]));
- q5bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[1], 4)), vreinterpretq_s8_u8(q5h.val[3]));
+ int32_t isum = 0;
- int32_t sumi1 = sc[0] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]));
- int32_t sumi2 = sc[1] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[1], q8bytes.val[1]));
- int32_t sumi3 = sc[2] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]));
- int32_t sumi4 = sc[3] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[3], q8bytes.val[3]));
+ for (int j = 0; j < QK_K/128; ++j) {
- sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);
- }
+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;
+ ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;
+ ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);
+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);
+ uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);
+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[1], 2);
+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+
+ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);
+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);
+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);
+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);
+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));
+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));
+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));
+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];
+
+ scale += 4;
+
+ q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
+
+ shifted = vshrq_n_u8(qhbits.val[0], 4);
+ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[1], 4);
+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[0], 6);
+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+ shifted = vshrq_n_u8(qhbits.val[1], 6);
+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
+
+ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);
+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);
+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);
+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);
+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));
+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));
+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));
+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));
+
+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +
+ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];
+ scale += 4;
+ }
+ //sum += isum * d_all * y[i].d;
+ sum += d_all * y[i].d * (isum - 32 * isum_mins);
- *s = sumf;
+ }
+ *s = sum;
#elif defined __AVX2__
const __m256i m4 = _mm256_set1_epi8(0xF);
- const __m256i mone = _mm256_set1_epi8(1);
+ const __m256i m2 = _mm256_set1_epi8(3);
+ const __m256i m32s = _mm256_set1_epi8(32);
__m256 acc = _mm256_setzero_ps();
for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q5 = x[i].qs;
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+
+ const uint8_t * restrict q4 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
const int8_t * restrict q8 = y[i].qs;
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);
- const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);
+ __m256i sumi = _mm256_setzero_si256();
- const __m256i scale_l = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[1]), _mm_set1_epi16(x[i].scales[0]));
- const __m256i scale_h = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[3]), _mm_set1_epi16(x[i].scales[2]));
+ int is = 0;
- int64_t aux64;
- memcpy(&aux64, x[i].qh, 8);
- const __m128i haux128 = _mm_set_epi64x(aux64 >> 1, aux64);
- const __m256i haux256 = MM256_SET_M128I(_mm_srli_epi16(haux128, 2), haux128);
+ for (int j = 0; j < QK_K/128; ++j) {
- const __m256i q5h_0 = _mm256_slli_epi16(_mm256_andnot_si256(haux256, mone), 4);
- const __m256i q5h_1 = _mm256_slli_epi16(_mm256_andnot_si256(_mm256_srli_epi16(haux256, 4), mone), 4);
+ const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));
+ const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));
+ const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));
+ const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));
+ is += 4;
- const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);
- const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);
+ const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
+ const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
+ const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+ const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);
+ const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);
+ const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);
+ const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);
- const __m256i p16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5l_0, q8_0));
- const __m256i p16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5l_1, q8_1));
- const __m256i s16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5h_0, q8_0));
- const __m256i s16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5h_1, q8_1));
+ const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);
+ const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);
+ const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);
+ const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);
+
+ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
+
+ __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);
+ __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);
+ __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);
+ __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);
+
+ __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);
+ __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);
+ __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);
+ __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);
+
+ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);
- const __m256i dot = _mm256_sub_epi32(_mm256_add_epi32(p16_0, p16_1), _mm256_add_epi32(s16_0, s16_1));
+ p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);
+ p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);
+ p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);
+ p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);
- acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(dot), acc);
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));
+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));
+
+ }
+ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
}
*s = hsum_float_8(acc);
@@ -5453,56 +7749,117 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r
#elif defined __AVX__
const __m128i m4 = _mm_set1_epi8(0xF);
- const __m128i mone = _mm_set1_epi8(1);
+ const __m128i m3 = _mm_set1_epi8(3);
+ const __m128i m32s = _mm_set1_epi8(32);
+ const __m128i m2 = _mm_set1_epi8(2);
__m256 acc = _mm256_setzero_ps();
for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q5 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5);
-
- const __m128i scale_0 = _mm_set1_epi16(x[i].scales[0]);
- const __m128i scale_1 = _mm_set1_epi16(x[i].scales[1]);
- const __m128i scale_2 = _mm_set1_epi16(x[i].scales[2]);
- const __m128i scale_3 = _mm_set1_epi16(x[i].scales[3]);
+ const uint8_t * restrict q4 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
+ const int8_t * restrict q8 = y[i].qs;
- int64_t aux64;
- memcpy(&aux64, x[i].qh, 8);
- const __m128i haux128_0 = _mm_set_epi64x(aux64 >> 1, aux64);
- const __m128i haux128_1 = _mm_srli_epi16(haux128_0, 2);
+ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);
- const __m128i q5h_0 = _mm_slli_epi16(_mm_andnot_si128(haux128_0, mone), 4);
- const __m128i q5h_1 = _mm_slli_epi16(_mm_andnot_si128(haux128_1, mone), 4);
- const __m128i q5h_2 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_0, 4), mone), 4);
- const __m128i q5h_3 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_1, 4), mone), 4);
+ __m128i sumi_0 = _mm_setzero_si128();
+ __m128i sumi_1 = _mm_setzero_si128();
- const __m128i q5l_0 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 0), m4);
- const __m128i q5l_1 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 1), m4);
- const __m128i q5l_2 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 0), 4), m4);
- const __m128i q5l_3 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 1), 4), m4);
+ __m128i shuffle = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
+ for (int j = 0; j < QK_K/128; ++j) {
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0));
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32));
+ const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;
+ const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;
+
+ const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);
+ const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);
+ const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 2), m3), 4);
+ const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 2), m3), 4);
+ const __m128i q4h_4 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 4), m3), 4);
+ const __m128i q4h_5 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 4), m3), 4);
+ const __m128i q4h_6 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 6), m3), 4);
+ const __m128i q4h_7 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 6), m3), 4);
+
+ const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+ const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
+
+ const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m4), q4h_0);
+ const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m4), q4h_1);
+ const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m4), q4h_2);
+ const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m4), q4h_3);
+ const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m4), q4h_4);
+ const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m4), q4h_5);
+ const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m4), q4h_6);
+ const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m4), q4h_7);
- const __m128i p16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5l_0, _mm256_extractf128_si256(q8_0, 0)));
- const __m128i p16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5l_1, _mm256_extractf128_si256(q8_0, 1)));
- const __m128i p16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5l_2, _mm256_extractf128_si256(q8_1, 0)));
- const __m128i p16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5l_3, _mm256_extractf128_si256(q8_1, 1)));
- const __m128i s16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5h_0, _mm256_extractf128_si256(q8_0, 0)));
- const __m128i s16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5h_1, _mm256_extractf128_si256(q8_0, 1)));
- const __m128i s16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5h_2, _mm256_extractf128_si256(q8_1, 0)));
- const __m128i s16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5h_3, _mm256_extractf128_si256(q8_1, 1)));
+ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i dot_0 = _mm_sub_epi32(_mm_add_epi32(p16_0, p16_2), _mm_add_epi32(s16_0, s16_2));
- const __m128i dot_1 = _mm_sub_epi32(_mm_add_epi32(p16_1, p16_3), _mm_add_epi32(s16_1, s16_3));
+ __m128i q8s_0 = _mm_maddubs_epi16(m32s, q8_0);
+ __m128i q8s_1 = _mm_maddubs_epi16(m32s, q8_1);
+ __m128i q8s_2 = _mm_maddubs_epi16(m32s, q8_2);
+ __m128i q8s_3 = _mm_maddubs_epi16(m32s, q8_3);
+ __m128i q8s_4 = _mm_maddubs_epi16(m32s, q8_4);
+ __m128i q8s_5 = _mm_maddubs_epi16(m32s, q8_5);
+ __m128i q8s_6 = _mm_maddubs_epi16(m32s, q8_6);
+ __m128i q8s_7 = _mm_maddubs_epi16(m32s, q8_7);
+
+ __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);
+ __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);
+ __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);
+ __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);
+ __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);
+ __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);
+ __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);
+ __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);
+
+ p16_0 = _mm_sub_epi16(p16_0, q8s_0);
+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);
+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);
+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);
+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);
+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);
+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);
+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);
+
+ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+ const __m128i scale_2 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+ const __m128i scale_3 = _mm_shuffle_epi8(scales, shuffle);
+ shuffle = _mm_add_epi8(shuffle, m2);
+
+ p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);
+ p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1);
+ p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);
+ p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3);
+ p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);
+ p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_2, scale_2)), p16_5);
+ p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);
+ p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_3, scale_3)), p16_7);
+
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));
+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));
+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(dot_1, dot_0))), acc);
+ }
+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);
}
*s = hsum_float_8(acc);
@@ -5510,68 +7867,83 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r
#elif defined __riscv_v_intrinsic
float sumf = 0;
-
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const int8_t * sc = x[i].scales;
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
- const uint8_t * restrict q5 = x[i].qs;
+ const uint8_t * restrict q6 = x[i].ql;
const uint8_t * restrict qh = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
+ const int8_t * restrict q8 = y[i].qs;
+
+ const int8_t * restrict scale = x[i].scales;
+
+ size_t vl;
vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
- // load qh
- vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(qh, 8);
- vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8));
+ int sum_t = 0;
+ int is = 0;
- size_t vl = 16;
+ for (int j = 0; j < QK_K/128; ++j) {
- // combine both qh_1 and qh_2
- vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl);
+ vl = 32;
- vuint8mf2_t qh_h0 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);
- vuint8mf2_t qh_h1 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), vl), 16, vl);
- vuint8mf2_t qh_h2 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(qh_x, vl), 16, vl);
- vuint8mf2_t qh_h3 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl);
+ // load qh
+ vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);
- vint8mf2_t qh_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h0);
- vint8mf2_t qh_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h1);
- vint8mf2_t qh_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h2);
- vint8mf2_t qh_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h3);
+ // load Q6
+ vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);
+ vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);
- // load q5
- vuint8mf2_t q5_x1 = __riscv_vle8_v_u8mf2(q5, vl);
- vuint8mf2_t q5_x2 = __riscv_vle8_v_u8mf2(q5+16, vl);
+ vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);
+ vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);
+ vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);
+ vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);
- vint8mf2_t q5s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x1, 0xF, vl));
- vint8mf2_t q5s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x2, 0xF, vl));
- vint8mf2_t q5s_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x1, 0x4, vl));
- vint8mf2_t q5s_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x2, 0x4, vl));
+ vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);
+ vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);
+ vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);
+ vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);
- vint8mf2_t q5_0 = __riscv_vsub_vv_i8mf2(q5s_0, qh_0, vl);
- vint8mf2_t q5_1 = __riscv_vsub_vv_i8mf2(q5s_1, qh_1, vl);
- vint8mf2_t q5_2 = __riscv_vsub_vv_i8mf2(q5s_2, qh_2, vl);
- vint8mf2_t q5_3 = __riscv_vsub_vv_i8mf2(q5s_3, qh_3, vl);
+ vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);
+ vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);
+ vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);
+ vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);
- // load Q8 and multiply it with Q5
- vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q5_0, __riscv_vle8_v_i8mf2(q8, vl), vl);
- vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q5_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl);
- vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q5_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl);
- vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q5_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl);
+ vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);
+ vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);
+ vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);
+ vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);
- vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl);
- vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl);
- vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl);
- vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl);
+ // load Q8 and take product
+ vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);
+ vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);
+ vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);
+ vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);
- int32_t sumi1 = sc[0] * __riscv_vmv_x_s_i32m1_i32(vs_0);
- int32_t sumi2 = sc[1] * __riscv_vmv_x_s_i32m1_i32(vs_1);
- int32_t sumi3 = sc[2] * __riscv_vmv_x_s_i32m1_i32(vs_2);
- int32_t sumi4 = sc[3] * __riscv_vmv_x_s_i32m1_i32(vs_3);
+ vl = 16;
- sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);
+ vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);
+ vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);
+ vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);
+ vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);
+ vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);
+ vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);
+ vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);
+ vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);
+
+ vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);
+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);
+ vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);
+ vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);
+
+ sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);
+
+ q6 += 64; qh += 32; q8 += 128; is=8;
+
+ }
+
+ sumf += d * sum_t;
}
@@ -5579,85 +7951,48 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r
#else
- int8_t aux8[QK_K];
- int16_t aux16[16];
+ int8_t aux8[QK_K];
+ int16_t aux16[8];
float sums [8];
+ int32_t aux32[8];
memset(sums, 0, 8*sizeof(float));
float sumf = 0;
for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q4 = x[i].qs;
- const uint8_t * restrict hm = x[i].qh;
+ const uint8_t * restrict q4 = x[i].ql;
+ const uint8_t * restrict qh = x[i].qh;
const int8_t * restrict q8 = y[i].qs;
+ memset(aux32, 0, 8*sizeof(int32_t));
int8_t * restrict a = aux8;
- for (int l = 0; l < 32; ++l) {
- a[l+ 0] = q4[l] & 0xF;
- a[l+32] = q4[l] >> 4;
- }
- for (int is = 0; is < 8; ++is) {
- uint8_t m = 1 << is;
- for (int l = 0; l < 8; ++l) a[8*is + l] -= (hm[l] & m ? 0 : 16);
+ for (int j = 0; j < QK_K; j += 128) {
+ for (int l = 0; l < 32; ++l) {
+ a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
+ a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
+ a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
+ a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
+ }
+ a += 128;
+ q4 += 64;
+ qh += 32;
}
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const int8_t * restrict sc = x[i].scales;
-
+ a = aux8;
+ int is = 0;
for (int j = 0; j < QK_K/16; ++j) {
- const float dl = d * sc[j];
- for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[8+l]);
- q8 += 16; a += 16;
+ int scale = x[i].scales[is++];
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
+ q8 += 8; a += 8;
}
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
}
for (int l = 0; l < 8; ++l) sumf += sums[l];
*s = sumf;
#endif
}
-#endif
-
-
-#if QK_K == 256
-
-#ifdef __x86_64__
-#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_default
-#include "ggml_vec_dot_q6_K_q8_K_256.inc"
-#undef ggml_vec_dot_q6_K_q8_K
-#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q6_K_q8_K_avx
-#ifndef __AVX__
-#define __UNDEF_AVX__
-#define __AVX__
-#endif
-#include "ggml_vec_dot_q6_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX__
-#undef __UNDEF_AVX__
-#undef __AVX__
-#endif
-#undef ggml_vec_dot_q6_K_q8_K
-#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q6_K_q8_K_avx2
-#ifndef __AVX2__
-#define __UNDEF_AVX2__
-#define __AVX2__
-#endif
-#include "ggml_vec_dot_q6_K_q8_K_256.inc"
-#ifdef __UNDEF_AVX2__
-#undef __UNDEF_AVX2__
-#undef __AVX2__
-#endif
-#undef ggml_vec_dot_q6_K_q8_K
-void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {
- ggml_vec_dot_q6_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);
- } else if (X86_HAVE(AVX)) {
- ggml_vec_dot_q6_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);
- } else {
- ggml_vec_dot_q6_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);
- }
-}
-#else
-#define static
-#include "ggml_vec_dot_q6_K_q8_K_256.inc"
-#undef static
-#endif
#else
@@ -10050,3 +12385,305 @@ void quantize_row_iq2_s(const float * restrict x, void * restrict vy, int64_t k)
block_iq2_s * restrict y = vy;
quantize_row_iq2_s_reference(x, y, k);
}
+
+static bool validate_float(float f, size_t i) {
+ if (isinf(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);
+ return false;
+ }
+
+ if (isnan(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);
+ return false;
+ }
+
+ return true;
+}
+
+static bool isinf_fp16(ggml_fp16_t f) {
+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) == 0;
+}
+
+static bool isnan_fp16(ggml_fp16_t f) {
+ return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) != 0;
+}
+
+static bool validate_fp16(ggml_fp16_t f, size_t i) {
+ if (isinf_fp16(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found inf value at block %zu\n", i);
+ return false;
+ }
+
+ if (isnan_fp16(f)) {
+ fprintf(stderr, "ggml_validate_row_data: found nan value at block %zu\n", i);
+ return false;
+ }
+
+ return true;
+}
+
+#define VALIDATE_ROW_DATA_D_F16_IMPL(type, data, nb) \
+ const type * q = (const type *) (data); \
+ for (size_t i = 0; i < (nb); ++i) { \
+ if (!validate_fp16(q[i].d, i)) { \
+ return false; \
+ } \
+ }
+
+#define VALIDATE_ROW_DATA_DM_F16_IMPL(type, data, nb, d, m) \
+ const type * q = (const type *) (data); \
+ for (size_t i = 0; i < (nb); ++i) { \
+ if (!validate_fp16(q[i].d, i) || !validate_fp16(q[i].m, i)) { \
+ return false; \
+ } \
+ }
+
+bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) {
+ if (type < 0 || type >= GGML_TYPE_COUNT) {
+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);
+ return false;
+ }
+
+ if (nbytes % ggml_type_size(type) != 0) {
+ fprintf(stderr, "%s: invalid size %zu for type %d\n", __func__, nbytes, type);
+ return false;
+ }
+
+ const size_t nb = nbytes/ggml_type_size(type);
+
+ switch (type) {
+ case GGML_TYPE_BF16:
+ {
+ int nans = 0;
+ int infs = 0;
+ const unsigned short * f = (const unsigned short *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ nans += (f[i] & 0x7fff) > 0x7f80;
+ infs += (f[i] & 0x7fff) == 0x7f80;
+ }
+ if (nans) {
+ fprintf(stderr, "%s: found %d NaNs in row of %zu BF16 values\n", __func__, nans, nb);
+ return false;
+ }
+ if (infs) {
+ fprintf(stderr, "%s: found %d infinities in row of %zu BF16 values\n", __func__, infs, nb);
+ return false;
+ }
+ } break;
+ case GGML_TYPE_F16:
+ {
+ const ggml_fp16_t * f = (const ggml_fp16_t *) data;
+ size_t i = 0;
+#if defined(__AVX2__)
+ for (; i + 15 < nb; i += 16) {
+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));
+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi16(0x7c00));
+ __m256i cmp = _mm256_cmpeq_epi16(vexp, _mm256_set1_epi16(0x7c00));
+ int mask = _mm256_movemask_epi8(cmp);
+ if (mask) {
+ for (size_t j = 0; j < 16; ++j) {
+ if (!validate_fp16(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#elif defined(__ARM_NEON)
+ for (; i + 7 < nb; i += 8) {
+ uint16x8_t v = vld1q_u16(f + i);
+ uint16x8_t vexp = vandq_u16(v, vdupq_n_u16(0x7c00));
+ uint16x8_t cmp = vceqq_u16(vexp, vdupq_n_u16(0x7c00));
+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(cmp, 4)), 0);
+ if (mask) {
+ for (size_t j = 0; j < 8; ++j) {
+ if (!validate_fp16(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#endif
+ for (; i < nb; ++i) {
+ if (!validate_fp16(f[i], i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_F32:
+ {
+ const float * f = (const float *) data;
+ size_t i = 0;
+#if defined(__AVX2__)
+ for (; i + 7 < nb; i += 8) {
+ __m256i v = _mm256_loadu_si256((const __m256i *)(f + i));
+ __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi32(0x7f800000));
+ __m256i cmp = _mm256_cmpeq_epi32(vexp, _mm256_set1_epi32(0x7f800000));
+ int mask = _mm256_movemask_epi8(cmp);
+ if (mask) {
+ for (size_t j = 0; j < 8; ++j) {
+ if (!validate_float(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#elif defined(__ARM_NEON)
+ for (; i + 3 < nb; i += 4) {
+ uint32x4_t v = vld1q_u32((const uint32_t *)f + i);
+ uint32x4_t vexp = vandq_u32(v, vdupq_n_u32(0x7f800000));
+ uint32x4_t cmp = vceqq_u32(vexp, vdupq_n_u32(0x7f800000));
+ uint64_t mask = vget_lane_u64(vreinterpret_u64_u16(vshrn_n_u32(cmp, 8)), 0);
+ if (mask) {
+ for (size_t j = 0; j < 4; ++j) {
+ if (!validate_float(f[i + j], i + j)) {
+ return false;
+ }
+ }
+ GGML_UNREACHABLE();
+ }
+ }
+#endif
+ for (; i < nb; ++i) {
+ if (!validate_float(f[i], i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_F64:
+ {
+ const double * f = (const double *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ if (!validate_float(f[i], i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_Q4_0:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb);
+ } break;
+ case GGML_TYPE_Q4_1:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_1, data, nb, d, m);
+ } break;
+ case GGML_TYPE_Q5_0:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_0, data, nb);
+ } break;
+ case GGML_TYPE_Q5_1:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_1, data, nb, d, m);
+ } break;
+ case GGML_TYPE_Q8_0:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q8_0, data, nb);
+ } break;
+ case GGML_TYPE_Q2_K:
+ {
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin);
+ } break;
+ case GGML_TYPE_Q3_K:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_K, data, nb);
+ } break;
+ case GGML_TYPE_Q4_K:
+ {
+ #ifdef GGML_QKK_64
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d[0], d[1]);
+ #else
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d, dmin);
+ #endif
+ } break;
+ case GGML_TYPE_Q5_K:
+ {
+ #ifdef GGML_QKK_64
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_K, data, nb);
+ #else
+ VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_K, data, nb, d, dmin);
+ #endif
+ } break;
+ case GGML_TYPE_Q6_K:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_q6_K, data, nb);
+ } break;
+ case GGML_TYPE_Q8_K:
+ {
+ const block_q8_K * q = (const block_q8_K *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ if (!validate_float(q[i].d, i)) {
+ return false;
+ }
+ }
+ } break;
+ case GGML_TYPE_IQ1_S:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb);
+ } break;
+ case GGML_TYPE_IQ1_M:
+ {
+ const block_iq1_m * q = (const block_iq1_m *) data;
+ for (size_t i = 0; i < nb; ++i) {
+ #if QK_K == 64
+ if (!validate_fp16(q[i].d, i)) {
+ return false;
+ }
+ #else
+ iq1m_scale_t scale;
+ const uint16_t * sc = (const uint16_t *)q[i].scales;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+ if (!validate_fp16(scale.f16, i)) {
+ return false;
+ }
+ #endif
+ }
+ } break;
+ case GGML_TYPE_IQ2_XXS:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xxs, data, nb);
+ } break;
+ case GGML_TYPE_IQ2_XS:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xs, data, nb);
+ } break;
+ case GGML_TYPE_IQ2_S:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_s, data, nb);
+ } break;
+ case GGML_TYPE_IQ3_XXS:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_xxs, data, nb);
+ } break;
+
+ case GGML_TYPE_IQ3_S:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_s, data, nb);
+ } break;
+ case GGML_TYPE_IQ4_XS:
+ #if QK_K != 64
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_xs, data, nb);
+ } break;
+ #endif
+ // with QK_K == 64, iq4_xs is iq4_nl
+ case GGML_TYPE_IQ4_NL:
+ {
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);
+ } break;
+ case GGML_TYPE_I8:
+ case GGML_TYPE_I16:
+ case GGML_TYPE_I32:
+ case GGML_TYPE_I64:
+ // nothing to validate
+ break;
+ default:
+ {
+ fprintf(stderr, "%s: invalid type %d\n", __func__, type);
+ return false;
+ }
+ }
+
+ return true;
+}
diff --git a/llama.cpp/ggml-quants.py b/llama.cpp/ggml-quants.py
new file mode 100644
index 00000000..c3d0a469
--- /dev/null
+++ b/llama.cpp/ggml-quants.py
@@ -0,0 +1,77 @@
+import re
+
+with open('llama.cpp/ggml-quants.h') as f:
+ prototypes = f.read()
+prototypes = [line.replace(';', '') for line in prototypes.split('\n')
+ if line.endswith(';') and not line.startswith('//')]
+FUNCS = [(re.search(r'(?<= )\w+', proto).group(0), proto)
+ for proto in prototypes]
+
+# BEGIN SPECIAL FUNCTIONS
+FUNCS.append(('ggml_validate_row_data', 'bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes)'))
+# END SPECIAL FUNCTIONS
+
+ARCHS = (
+ ('amd_avx512', '__x86_64__', ('FMA', 'F16C', 'AVX2', 'AVX512F')),
+ ('amd_avx2', '__x86_64__', ('FMA', 'F16C', 'AVX2')),
+ ('amd_avx', '__x86_64__', ()),
+ ('arm80', '__aarch64__', ()),
+)
+
+for arch, mac, needs in ARCHS:
+ path = 'llama.cpp/ggml-quants-%s.c' % (arch.replace('_', '-'))
+ with open(path, 'w') as f:
+ f.write('#ifdef %s\n' % (mac))
+ for func, proto in FUNCS:
+ f.write('#define %s %s_%s\n' % (func, func, arch))
+ f.write('#include "ggml-quants.inc"\n')
+ f.write('#endif // %s\n' % (mac))
+
+with open('llama.cpp/ggml-quants.cpp', 'w') as f:
+ f.write('#include <cosmo.h>\n')
+ f.write('#include <sys/auxv.h>\n')
+ f.write('#include <libc/sysv/consts/hwcap.h>\n')
+ f.write('#include "ggml-quants.h"\n')
+ f.write('\n')
+ for func, proto in FUNCS:
+ for arch, mac, needs in ARCHS:
+ f.write('extern "C" %s;\n' % (proto.replace(func, '%s_%s' % (func, arch))))
+ f.write('\n')
+ f.write('static const struct QuantFuncs {\n')
+ for func, proto in FUNCS:
+ f.write(' typeof(%s) *ptr_%s;\n' % (func, func))
+ f.write('\n')
+ f.write(' QuantFuncs() {\n')
+ f.write('#ifdef __x86_64__\n')
+ for arch, mac, needs in ARCHS:
+ if mac == '__x86_64__':
+ f.write(' if (%s) {\n' % (' && '.join('X86_HAVE(%s)' % (need) for need in needs) or '1'))
+ for func, proto in FUNCS:
+ f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))
+ f.write(' return;\n')
+ f.write(' }\n')
+ f.write('#else\n')
+ for func, proto in FUNCS:
+ f.write(' ptr_%s = %s_arm80;\n' % (func, func))
+ f.write('#endif\n')
+ f.write(' }\n')
+ f.write('} funcs;\n')
+ f.write('\n')
+ for func, proto in FUNCS:
+ proto = proto.replace(';', '')
+ f.write(proto + ' {\n')
+ if 'imatrix' in proto:
+ args = 'src, dst, nrows, n_per_row, imatrix'
+ elif 'quantize' in proto:
+ args = 'x, y, k'
+ elif 'vec_dot' in proto:
+ args = 'n, s, bs, vx, bx, vy, by, nrc'
+ elif 'grid_size' in proto:
+ args = 'grid_size'
+ elif 'validate' in proto:
+ args = 'type, data, nbytes'
+ else:
+ args = 'type'
+ f.write(' return funcs.ptr_%s(%s);\n' % (func, args))
+ f.write('}\n')
+ f.write('\n')
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 37715f20..371f738a 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -117,6 +117,7 @@ void ggml_print_backtrace(void) {
/*#define GGML_PERF*/
#define GGML_DEBUG 0
+#define GGML_KAHAN 0
#define GGML_SOFT_MAX_UNROLL 4
#define GGML_VEC_DOT_UNROLL 2
@@ -956,7 +957,7 @@ do { \
#define GGML_F16_VEC_ZERO GGML_F16x8_ZERO
#define GGML_F16_VEC_SET1 GGML_F16x8_SET1
#define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)
- #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE(p, r[i])
+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((ggml_fp16_internal_t *)(p), r[i])
#define GGML_F16_VEC_FMA GGML_F16x8_FMA
#define GGML_F16_VEC_ADD GGML_F16x8_ADD
#define GGML_F16_VEC_SUB GGML_F16x8_SUB
@@ -984,7 +985,7 @@ do { \
#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO
#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1
#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)
- #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])
+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((ggml_fp16_internal_t *)(p), r[i])
#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
#define GGML_F16_VEC_SUB GGML_F32Cx4_SUB
@@ -1056,7 +1057,7 @@ do { \
// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F
// so F16C guard isn't required
-#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(x)))
+#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))
#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))
#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)
@@ -1140,10 +1141,10 @@ do { \
#if defined(__F16C__)
// the _mm256_cvt intrinsics require F16C
-#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(x)))
+#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))
#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))
#else
-static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {
+static inline __m256 __avx_f32cx8_load(const ggml_fp16_t *x) {
float tmp[8];
for (int i = 0; i < 8; i++) {
@@ -1613,10 +1614,14 @@ static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t *
ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[j]);
- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[j], ky);
- err[j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[j]), ky);
- sum[j] = kt;
+ if (GGML_KAHAN) {
+ GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[j]);
+ GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[j], ky);
+ err[j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[j]), ky);
+ sum[j] = kt;
+ } else {
+ sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);
+ }
}
}
@@ -1663,10 +1668,14 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * re
for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {
ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);
- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[k][j]);
- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[k][j], ky);
- err[k][j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[k][j]), ky);
- sum[k][j] = kt;
+ if (GGML_KAHAN) {
+ GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[k][j]);
+ GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[k][j], ky);
+ err[k][j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[k][j]), ky);
+ sum[k][j] = kt;
+ } else {
+ sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);
+ }
}
}
}
@@ -1726,6 +1735,37 @@ inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float
#endif
}
+inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
+
+ GGML_F16_VEC ax[GGML_F16_ARR];
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);
+
+ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
+ }
+#endif
+}
+
// xs and vs are byte strides of x and v
inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {
@@ -1810,6 +1850,35 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {
#endif
}
+inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
+
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_MUL(ay[j], vx);
+
+ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
+ }
+#endif
+}
+
inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }
inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }
inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }
@@ -1828,39 +1897,6 @@ inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float
////////////////////////////////////////////////////////////////////////////////
// SYNCHRONIZATION PRIMITIVES
-struct ggml_barrier {
- int nth;
- atomic_uint phase;
- atomic_uint count;
-};
-
-int ggml_delay(int backoff) {
- if (backoff < 13) {
- volatile int i;
- for (i = 0; i != 1 << backoff; i++) {
- }
- backoff++;
- } else {
- sched_yield();
- }
- return backoff;
-}
-
-// creates barrier and blocks until all threads call this
-void ggml_syncthreads(struct ggml_compute_params * params) {
- struct ggml_barrier * b = params->barrier;
- unsigned phase = atomic_load_explicit(&b->phase, memory_order_relaxed);
- if (atomic_fetch_add_explicit(&b->count, 1, memory_order_acq_rel) + 1 == b->nth) {
- atomic_store_explicit(&b->count, 0, memory_order_relaxed);
- atomic_store_explicit(&b->phase, phase + 1, memory_order_release);
- } else {
- int backoff = 0;
- while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase) {
- backoff = ggml_delay(backoff);
- }
- }
-}
-
void ggml_once(atomic_uint * once, void init(void)) {
uint32_t old = atomic_load_explicit(once, memory_order_acquire);
if (!old && atomic_compare_exchange_strong_explicit(once, &old, 1,
@@ -2104,6 +2140,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
"LEAKY_RELU",
"FLASH_ATTN",
+ "FLASH_ATTN_EXT",
"FLASH_FF",
"FLASH_ATTN_BACK",
"SSM_CONV",
@@ -2130,7 +2167,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
"CROSS_ENTROPY_LOSS_BACK",
};
-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");
+static_assert(GGML_OP_COUNT == 77, "GGML_OP_COUNT != 77");
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"none",
@@ -2194,6 +2231,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"leaky_relu(x)",
"flash_attn(x)",
+ "flash_attn_ext(x)",
"flash_ff(x)",
"flash_attn_back(x)",
"ssm_conv(x)",
@@ -2220,7 +2258,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"cross_entropy_loss_back(x,y)",
};
-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");
+static_assert(GGML_OP_COUNT == 77, "GGML_OP_COUNT != 77");
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
@@ -2716,7 +2754,7 @@ GGML_CALL bool ggml_is_empty(const struct ggml_tensor * tensor) {
return false;
}
-bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+GGML_CALL bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
return
@@ -4668,6 +4706,8 @@ struct ggml_tensor * ggml_mul_mat(
void ggml_mul_mat_set_prec(
struct ggml_tensor * a,
enum ggml_prec prec) {
+ GGML_ASSERT(a->op == GGML_OP_MUL_MAT);
+
const int32_t prec_i32 = (int32_t) prec;
ggml_set_op_params_i32(a, 0, prec_i32);
@@ -5506,17 +5546,23 @@ static struct ggml_tensor * ggml_soft_max_impl(
GGML_ASSERT(ggml_is_contiguous(a));
if (mask) {
+ GGML_ASSERT(mask->type == GGML_TYPE_F16 || mask->type == GGML_TYPE_F32);
GGML_ASSERT(ggml_is_contiguous(mask));
GGML_ASSERT(ggml_is_matrix(mask));
- GGML_ASSERT(ggml_can_repeat_rows(mask, a));
+ GGML_ASSERT(mask->ne[0] == a->ne[0]);
+ GGML_ASSERT(mask->ne[1] >= a->ne[1]);
}
if (pos) {
GGML_ASSERT(ggml_is_vector(pos));
- GGML_ASSERT(pos->type == GGML_TYPE_F32);
+ GGML_ASSERT(pos->type == GGML_TYPE_F16 || pos->type == GGML_TYPE_F32);
GGML_ASSERT(pos->ne[0] == a->ne[0]);
}
+ if (pos && mask) {
+ GGML_ASSERT(pos->type == mask->type);
+ }
+
if (max_bias > 0.0f) {
GGML_ASSERT(pos);
}
@@ -6325,6 +6371,59 @@ struct ggml_tensor * ggml_flash_attn(
return result;
}
+// ggml_flash_attn_ext
+
+struct ggml_tensor * ggml_flash_attn_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * mask,
+ float scale) {
+ GGML_ASSERT(ggml_can_mul_mat(k, q));
+ // TODO: check if vT can be multiplied by (k*qT)
+ if (mask) {
+ GGML_ASSERT(ggml_is_contiguous(mask));
+ GGML_ASSERT(mask->ne[2] == 1);
+ GGML_ASSERT(mask->ne[3] == 1);
+ GGML_ASSERT(mask->ne[1] >= GGML_PAD(q->ne[1], GGML_KQ_MASK_PAD) &&
+ "the Flash-Attention kernel requires the mask to be padded to GGML_KQ_MASK_PAD and at least n_queries big");
+ //GGML_ASSERT(ggml_can_repeat_rows(mask, qk));
+ }
+
+ bool is_node = false;
+
+ if (q->grad || k->grad || v->grad) {
+ is_node = true;
+ }
+
+ // permute(0, 2, 1, 3)
+ int64_t ne[4] = { q->ne[0], q->ne[2], q->ne[1], q->ne[3] };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ float params[] = { scale };
+ ggml_set_op_params(result, params, sizeof(params));
+
+ result->op = GGML_OP_FLASH_ATTN_EXT;
+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
+ result->src[0] = q;
+ result->src[1] = k;
+ result->src[2] = v;
+ result->src[3] = mask;
+
+ return result;
+}
+
+void ggml_flash_attn_ext_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec) {
+ GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT);
+
+ const int32_t prec_i32 = (int32_t) prec;
+
+ ggml_set_op_params_i32(a, 1, prec_i32); // scale is on first pos
+}
+
// ggml_flash_ff
struct ggml_tensor * ggml_flash_ff(
@@ -9609,8 +9708,8 @@ static void ggml_compute_forward_concat_f32(
}
static void ggml_compute_forward_concat(
- struct ggml_compute_params * params,
- struct ggml_tensor * dst) {
+ struct ggml_compute_params* params,
+ struct ggml_tensor* dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10936,7 +11035,8 @@ static void ggml_compute_forward_mul_mat(
params->type,
src0->type,
src1->type,
- dst->type))
+ dst->type,
+ dst->op_params[0]))
goto UseGgmlGemm1;
return;
}
@@ -10951,11 +11051,11 @@ UseGgmlGemm1:;
assert(params->wsize >= ne11*ne12*ne13*row_size);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
- unsigned chore = 0;
+ unsigned chore = 0; // [jart] need for speed
for (int64_t i13 = 0; i13 < ne13; ++i13) {
for (int64_t i12 = 0; i12 < ne12; ++i12) {
for (int64_t i11 = 0; i11 < ne11; ++i11) {
- if (chore++ % nth == ith) {
+ if (chore++ % nth == ith) { // [jart] need for speed
from_float_to_vec_dot((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), (void *) wdata, ne10);
}
wdata += row_size;
@@ -10975,15 +11075,13 @@ UseGgmlGemm1:;
const size_t row_size = ggml_row_size(vec_dot_type, ne10);
#if GGML_USE_LLAMAFILE
- if (src1_cont) {
+ if (src1->type != vec_dot_type) {
for (int64_t i13 = 0; i13 < ne13; i13++)
for (int64_t i12 = 0; i12 < ne12; i12++)
if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type),
(const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
nb01/ggml_type_size(src0->type),
- (const char *)wdata + ggml_row_size(vec_dot_type,
- nb12/ggml_type_size(src1->type)*i12 +
- nb13/ggml_type_size(src1->type)*i13),
+ (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,
row_size/ggml_type_size(vec_dot_type),
(char *)dst->data + i12*nb2 + i13*nb3,
nb1/ggml_type_size(dst->type),
@@ -10991,7 +11089,8 @@ UseGgmlGemm1:;
params->type,
src0->type,
vec_dot_type,
- dst->type))
+ dst->type,
+ dst->op_params[0]))
goto UseGgmlGemm2;
return;
}
@@ -11092,11 +11191,11 @@ UseGgmlGemm2:;
}
#if 0
- fprintf(stderr, "MULMAT %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) * %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) -> %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) vdt=%s\n",
+ fprintf(stderr, "MULMAT %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) * %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) -> %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) vdt=%s src1_cont=%d\n",
ggml_type_name(src0->type), ne00, ne01, ne02, nb00, nb01, nb02,
ggml_type_name(src1->type), ne10, ne11, ne12, nb10, nb11, nb12,
ggml_type_name(src0->type), ne0, ne1, ne2, nb0, nb1, nb2,
- ggml_type_name(vec_dot_type));
+ ggml_type_name(vec_dot_type), src1_cont);
#endif
}
@@ -11113,11 +11212,11 @@ static void ggml_compute_forward_mul_mat_id(
if (llamafile_mixmul(params, src0, src1, ids, dst))
return;
+ GGML_TENSOR_BINARY_OP_LOCALS
+
const int ith = params->ith;
const int nth = params->nth;
- GGML_TENSOR_BINARY_OP_LOCALS
-
const enum ggml_type type = src0->type;
const bool src1_cont = ggml_is_contiguous(src1);
@@ -12424,7 +12523,7 @@ static void ggml_compute_forward_soft_max_f32(
GGML_TENSOR_UNARY_OP_LOCALS
- const int64_t ne11 = src1 ? src1->ne[1] : 1;
+ //const int64_t ne11 = src1 ? src1->ne[1] : 1;
// TODO: is this supposed to be ceil instead of floor?
// https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370
@@ -12447,19 +12546,31 @@ static void ggml_compute_forward_soft_max_f32(
float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;
// when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching
- float * pos = src2 ? (float *) src2->data : src0->data;
+ ggml_fp16_t * pos_f16 = src2 ? (ggml_fp16_t *) src2->data : src0->data;
+ float * pos_f32 = src2 ? (float *) src2->data : src0->data;
+
+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);
for (int i1 = ir0; i1 < ir1; i1++) {
float * sp = (float *)((char *) src0->data + i1*src0->nb[1]);
float * dp = (float *)((char *) dst->data + i1*dst->nb[1]);
// broadcast the mask across rows
- float * mp = src1 ? (float *)((char *) src1->data + (i1%ne11)*src1->nb[1]) : NULL;
+ ggml_fp16_t * mp_f16 = src1 ? (ggml_fp16_t *)((char *) src1->data) + (i1%ne01)*ne00 : NULL;
+ float * mp_f32 = src1 ? (float *)((char *) src1->data) + (i1%ne01)*ne00 : NULL;
ggml_vec_cpy_f32 (nc, wp, sp);
ggml_vec_scale_f32(nc, wp, scale);
- if (mp) {
- ggml_vec_acc_f32(nc, wp, mp);
+ if (mp_f32) {
+ if (use_f16) {
+ for (int i = 0; i < nc; ++i) {
+ wp[i] += GGML_FP16_TO_FP32(mp_f16[i]);
+ }
+ } else {
+ for (int i = 0; i < nc; ++i) {
+ wp[i] += mp_f32[i];
+ }
+ }
}
// ALiBi bias
@@ -12467,8 +12578,14 @@ static void ggml_compute_forward_soft_max_f32(
const uint32_t h = (i1/ne01)%ne02; // head
const float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1);
- for (int i = 0; i < nc; i++) {
- wp[i] = wp[i] + slope*pos[i];
+ if (use_f16) {
+ for (int i = 0; i < nc; ++i) {
+ wp[i] += slope*GGML_FP16_TO_FP32(pos_f16[i]);
+ }
+ } else {
+ for (int i = 0; i < nc; ++i) {
+ wp[i] += slope*pos_f32[i];
+ }
}
}
@@ -14715,6 +14832,198 @@ static void ggml_compute_forward_flash_attn(
}
}
+// ggml_compute_forward_flash_attn_ext
+
+static void ggml_compute_forward_flash_attn_ext_f16(
+ struct ggml_compute_params * params,
+ const struct ggml_tensor * q,
+ const struct ggml_tensor * k,
+ const struct ggml_tensor * v,
+ const struct ggml_tensor * mask,
+ struct ggml_tensor * dst) {
+ int64_t t0 = ggml_perf_time_us();
+ UNUSED(t0);
+
+ GGML_TENSOR_LOCALS(int64_t, neq, q, ne)
+ GGML_TENSOR_LOCALS(size_t, nbq, q, nb)
+ GGML_TENSOR_LOCALS(int64_t, nek, k, ne)
+ GGML_TENSOR_LOCALS(size_t, nbk, k, nb)
+ GGML_TENSOR_LOCALS(int64_t, nev, v, ne)
+ GGML_TENSOR_LOCALS(size_t, nbv, v, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t D = neq0;
+ const int64_t N = neq1;
+
+ GGML_ASSERT(ne0 == D);
+ GGML_ASSERT(ne2 == N);
+
+ GGML_ASSERT(nbq0 == sizeof(float));
+ GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));
+
+ GGML_ASSERT(neq0 == D);
+ GGML_ASSERT(nek0 == D);
+ GGML_ASSERT(nev0 == D);
+
+ GGML_ASSERT(neq1 == N);
+ GGML_ASSERT(nev0 == D);
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ // broadcast factors
+ const int64_t rk2 = neq2/nek2;
+ const int64_t rk3 = neq3/nek3;
+
+ const int64_t rv2 = neq2/nev2;
+ const int64_t rv3 = neq3/nev3;
+
+ if (params->type == GGML_TASK_TYPE_INIT) {
+ return;
+ }
+
+ if (params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ // parallelize by q rows using ggml_vec_dot_f32
+
+ // total rows in q
+ const int nr = neq1*neq2*neq3;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ float scale = 1.0f;
+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
+
+ // loop over n_batch and n_head
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // q indices
+ const int iq3 = ir/(neq2*neq1);
+ const int iq2 = (ir - iq3*neq2*neq1)/neq1;
+ const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
+
+ float S = 0.0f;
+ float M = -INFINITY;
+
+ float * V32 = (float *) params->wdata + ith*(2*D + CACHE_LINE_SIZE_F32);
+ ggml_fp16_t * Q16 = (ggml_fp16_t *) (V32); // reuse memory
+ ggml_fp16_t * V16 = (ggml_fp16_t *) (V32 + D);
+
+ memset(V16, 0, D*sizeof(ggml_fp16_t));
+
+ const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1]) : NULL;
+
+ // k indices
+ const int ik3 = iq3 / rk3;
+ const int ik2 = iq2 / rk2;
+
+ // v indices
+ const int iv3 = iq3 / rv3;
+ const int iv2 = iq2 / rv2;
+
+ // online softmax / attention
+ // loop over n_kv and n_head_kv
+ // ref: https://arxiv.org/pdf/2112.05682.pdf
+ for (int64_t ic = 0; ic < nek1; ++ic) {
+ const float mv = mp ? GGML_FP16_TO_FP32(mp[ic]) : 0.0f;
+ if (mv == -INFINITY) {
+ continue;
+ }
+
+ float s;
+
+ // convert Q to F16 in V32
+ {
+ const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));
+
+ for (int64_t d = 0; d < D; ++d) {
+ Q16[d] = GGML_FP32_TO_FP16(pq[d]);
+ }
+ }
+
+ ggml_vec_dot_f16(D,
+ &s, 0,
+ (ggml_fp16_t *) ((char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,
+ Q16, 0, 1);
+
+ s = s*scale + mv;
+
+ const float Mold = M;
+
+ float ms = 1.0f;
+ float vs = 1.0f;
+
+ if (s > M) {
+ M = s;
+ ms = expf(Mold - M);
+
+ // V = V*expf(Mold - M)
+ ggml_vec_scale_f16(D, V16, ms);
+ } else {
+ vs = expf(s - M);
+ }
+
+ const ggml_fp16_t * v16 = (const ggml_fp16_t *) ((char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
+
+ // V += v*expf(s - M)
+ ggml_vec_mad_f16(D, V16, v16, vs);
+
+ S = S*ms + vs;
+ }
+
+ // V /= S
+ for (int64_t d = 0; d < D; ++d) {
+ V32[d] = GGML_FP16_TO_FP32(V16[d])/S;
+ }
+
+ // dst indices
+ const int i1 = iq1;
+ const int i2 = iq2;
+ const int i3 = iq3;
+
+ // original
+ //memcpy((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3), V, nev0*sizeof(float));
+
+ // permute(0, 2, 1, 3)
+ memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, V32, nb1);
+ }
+}
+
+static void ggml_compute_forward_flash_attn_ext(
+ struct ggml_compute_params * params,
+ const struct ggml_tensor * q,
+ const struct ggml_tensor * k,
+ const struct ggml_tensor * v,
+ const struct ggml_tensor * mask,
+ struct ggml_tensor * dst) {
+ switch (dst->op_params[1]) {
+ case GGML_PREC_DEFAULT:
+ case GGML_PREC_F32:
+ {
+ // uses F32 accumulators
+ ggml_compute_forward_flash_attn_ext_f16(params, q, k, v, mask, dst);
+ } break;
+ default:
+ {
+ GGML_ASSERT(false);
+ } break;
+ }
+}
+
// ggml_compute_forward_flash_ff
static void ggml_compute_forward_flash_ff_f16(
@@ -16494,6 +16803,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
const bool masked = t != 0;
ggml_compute_forward_flash_attn(params, masked, tensor);
} break;
+ case GGML_OP_FLASH_ATTN_EXT:
+ {
+ ggml_compute_forward_flash_attn_ext(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], tensor);
+ } break;
case GGML_OP_FLASH_FF:
{
ggml_compute_forward_flash_ff(params, tensor);
@@ -17506,6 +17819,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
GGML_ASSERT(false); // TODO: not implemented
} break;
case GGML_OP_FLASH_ATTN:
+ case GGML_OP_FLASH_ATTN_EXT:
{
struct ggml_tensor * flash_grad = NULL;
if (src0->grad || src1->grad || tensor->src[2]->grad) {
@@ -18278,6 +18592,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_
n_tasks = n_threads;
} break;
case GGML_OP_FLASH_ATTN:
+ case GGML_OP_FLASH_ATTN_EXT:
{
n_tasks = n_threads;
} break;
@@ -18702,6 +19017,12 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2
}
} break;
+ case GGML_OP_FLASH_ATTN_EXT:
+ {
+ const int64_t ne00 = node->src[0]->ne[0]; // D
+
+ cur = 2*sizeof(float)*ne00*n_tasks; // 2x head size
+ } break;
case GGML_OP_FLASH_FF:
{
if (node->src[1]->type == GGML_TYPE_F32) {
@@ -20775,7 +21096,7 @@ static void gguf_free_kv(struct gguf_kv * kv) {
}
struct gguf_context * gguf_init_empty(void) {
- struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));
+ struct gguf_context * ctx = GGML_CALLOC(1, sizeof(struct gguf_context));
memcpy(ctx->header.magic, GGUF_MAGIC, sizeof(ctx->header.magic));
ctx->header.version = GGUF_VERSION;
@@ -20815,7 +21136,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
bool ok = true;
- struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));
+ struct gguf_context * ctx = GGML_CALLOC(1, sizeof(struct gguf_context));
// read the header
{
@@ -20850,9 +21171,13 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
// read the kv pairs
{
- ctx->kv = GGML_MALLOC(ctx->header.n_kv * sizeof(struct gguf_kv));
+ const uint64_t n_kv = ctx->header.n_kv;
- for (uint64_t i = 0; i < ctx->header.n_kv; ++i) {
+ // header.n_kv will hold the actual value of pairs that were successfully read in the loop below
+ ctx->header.n_kv = 0;
+ ctx->kv = GGML_CALLOC(n_kv, sizeof(struct gguf_kv));
+
+ for (uint64_t i = 0; i < n_kv; ++i) {
struct gguf_kv * kv = &ctx->kv[i];
//fprintf(stderr, "%s: reading kv %d\n", __func__, i);
@@ -20900,7 +21225,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
return NULL;
}
- kv->value.arr.data = GGML_MALLOC(kv->value.arr.n * gguf_type_size(kv->value.arr.type));
+ kv->value.arr.data = GGML_CALLOC(kv->value.arr.n, gguf_type_size(kv->value.arr.type));
ok = ok && gguf_fread_el(file, kv->value.arr.data, kv->value.arr.n * gguf_type_size(kv->value.arr.type), &offset);
} break;
@@ -20913,7 +21238,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
return NULL;
}
- kv->value.arr.data = GGML_MALLOC(kv->value.arr.n * sizeof(struct gguf_str));
+ kv->value.arr.data = GGML_CALLOC(kv->value.arr.n, sizeof(struct gguf_str));
for (uint64_t j = 0; j < kv->value.arr.n; ++j) {
ok = ok && gguf_fread_str(file, &((struct gguf_str *) kv->value.arr.data)[j], &offset);
@@ -20929,6 +21254,8 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
if (!ok) {
break;
}
+
+ ctx->header.n_kv++;
}
if (!ok) {
@@ -20940,7 +21267,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
// read the tensor infos
{
- ctx->infos = GGML_MALLOC(ctx->header.n_tensors * sizeof(struct gguf_tensor_info));
+ ctx->infos = GGML_CALLOC(ctx->header.n_tensors, sizeof(struct gguf_tensor_info));
for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {
struct gguf_tensor_info * info = &ctx->infos[i];
@@ -20961,8 +21288,17 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
ok = ok && gguf_fread_el (file, &info->type, sizeof(info->type), &offset);
ok = ok && gguf_fread_el (file, &info->offset, sizeof(info->offset), &offset);
+ // TODO: return an error instead of crashing with GGML_ASSERT
gguf_tensor_info_sanitize(info);
+ // make sure there is no duplicated tensor names
+ for (uint64_t j = 0; j < i; ++j) {
+ if (strcmp(info->name.data, ctx->infos[j].name.data) == 0) {
+ fprintf(stderr, "%s: duplicated tensor name %s\n", __func__, info->name.data);
+ ok = false;
+ }
+ }
+
if (!ok) {
fprintf(stderr, "%s: failed to read tensor info\n", __func__);
gguf_free(ctx);
@@ -21125,7 +21461,7 @@ void gguf_free(struct gguf_context * ctx) {
GGML_FREE(ctx->infos);
}
- GGML_ALIGNED_FREE(ctx);
+ GGML_FREE(ctx);
}
const char * gguf_type_name(enum gguf_type type) {
@@ -21436,7 +21772,7 @@ void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_ty
ctx->kv[idx].type = GGUF_TYPE_ARRAY;
ctx->kv[idx].value.arr.type = type;
ctx->kv[idx].value.arr.n = n;
- ctx->kv[idx].value.arr.data = GGML_MALLOC(n*gguf_type_size(type));
+ ctx->kv[idx].value.arr.data = GGML_CALLOC(n, gguf_type_size(type));
memcpy(ctx->kv[idx].value.arr.data, data, n*gguf_type_size(type));
}
@@ -21446,7 +21782,7 @@ void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char **
ctx->kv[idx].type = GGUF_TYPE_ARRAY;
ctx->kv[idx].value.arr.type = GGUF_TYPE_STRING;
ctx->kv[idx].value.arr.n = n;
- ctx->kv[idx].value.arr.data = GGML_MALLOC(n*sizeof(struct gguf_str));
+ ctx->kv[idx].value.arr.data = GGML_CALLOC(n, sizeof(struct gguf_str));
for (int i = 0; i < n; i++) {
struct gguf_str * str = &((struct gguf_str *)ctx->kv[idx].value.arr.data)[i];
str->n = strlen(data[i]);
@@ -21473,7 +21809,7 @@ void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {
case GGUF_TYPE_ARRAY:
{
if (src->kv[i].value.arr.type == GGUF_TYPE_STRING) {
- const char ** data = GGML_MALLOC(src->kv[i].value.arr.n*sizeof(char *));
+ const char ** data = GGML_CALLOC(src->kv[i].value.arr.n, sizeof(char *));
for (uint32_t j = 0; j < src->kv[i].value.arr.n; j++) {
data[j] = ((struct gguf_str *)src->kv[i].value.arr.data)[j].data;
}
@@ -21493,6 +21829,10 @@ void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {
void gguf_add_tensor(
struct gguf_context * ctx,
const struct ggml_tensor * tensor) {
+ if (gguf_find_tensor(ctx, tensor->name) != -1) {
+ GGML_ASSERT(false && "duplicated tensor name");
+ }
+
const int idx = ctx->header.n_tensors;
ctx->infos = realloc(ctx->infos, (idx + 1)*sizeof(struct gguf_tensor_info));
@@ -21561,7 +21901,7 @@ struct gguf_buf {
static struct gguf_buf gguf_buf_init(size_t size) {
struct gguf_buf buf = {
- /*buf.data =*/ size == 0 ? NULL : GGML_MALLOC(size),
+ /*buf.data =*/ size == 0 ? NULL : GGML_CALLOC(1, size),
/*buf.size =*/ size,
/*buf.offset =*/ 0,
};
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index d9aa1eb8..b45c29ba 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -479,6 +479,7 @@ extern "C" {
GGML_OP_LEAKY_RELU,
GGML_OP_FLASH_ATTN,
+ GGML_OP_FLASH_ATTN_EXT,
GGML_OP_FLASH_FF,
GGML_OP_FLASH_ATTN_BACK,
GGML_OP_SSM_CONV,
@@ -683,14 +684,8 @@ extern "C" {
// work buffer for all threads
size_t wsize;
void * wdata;
-
- struct ggml_barrier * barrier;
};
- GGML_API void ggml_syncthreads (struct ggml_compute_params *);
- GGML_API void * ggml_acquire_wdata(struct ggml_compute_params *);
- GGML_API void ggml_release_wdata(struct ggml_compute_params *);
-
// numa strategies
enum ggml_numa_strategy {
GGML_NUMA_STRATEGY_DISABLED = 0,
@@ -767,11 +762,13 @@ extern "C" {
GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);
GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars
- GGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
+ GGML_API GGML_CALL bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
// use this to compute the memory overhead of a tensor
GGML_API size_t ggml_tensor_overhead(void);
+ GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
+
// main
GGML_API struct ggml_context * ggml_init(struct ggml_init_params params);
@@ -1730,6 +1727,25 @@ extern "C" {
struct ggml_tensor * v,
bool masked);
+#define GGML_KQ_MASK_PAD 32
+
+ // q: [n_embd, n_batch, n_head, 1]
+ // k: [n_embd, n_kv, n_head_kv, 1]
+ // v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!
+ // mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = GGML_PAD(n_batch, GGML_KQ_MASK_PAD) !!
+ // res: [n_embd, n_head, n_batch, 1] !! permuted !!
+ GGML_API struct ggml_tensor * ggml_flash_attn_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * mask,
+ float scale);
+
+ GGML_API void ggml_flash_attn_ext_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec);
+
GGML_API struct ggml_tensor * ggml_flash_attn_back(
struct ggml_context * ctx,
struct ggml_tensor * q,
diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc
deleted file mode 100644
index 3c675205..00000000
--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc
+++ /dev/null
@@ -1,522 +0,0 @@
-static void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- assert(n % QK_K == 0);
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
- const uint32_t kmask1 = 0x03030303;
- const uint32_t kmask2 = 0x0f0f0f0f;
-
- const block_q3_K * restrict x = vx;
- const block_q8_K * restrict y = vy;
-
- const int nb = n / QK_K;
-
-#ifdef __ARM_NEON
-
- uint32_t aux[3];
- uint32_t utmp[4];
-
- const uint8x16_t m3b = vdupq_n_u8(0x3);
- const int32x4_t vzero = vdupq_n_s32(0);
-
- const uint8x16_t m0 = vdupq_n_u8(1);
- const uint8x16_t m1 = vshlq_n_u8(m0, 1);
- const uint8x16_t m2 = vshlq_n_u8(m0, 2);
- const uint8x16_t m3 = vshlq_n_u8(m0, 3);
- const int8_t m32 = 32;
-
- ggml_int8x16x4_t q3bytes;
-
- float sum = 0;
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
-
- const uint8_t * restrict q3 = x[i].qs;
- const uint8_t * restrict qh = x[i].hmask;
- const int8_t * restrict q8 = y[i].qs;
-
- ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);
-
- ggml_uint8x16x4_t q3h;
-
- int32_t isum = 0;
-
- // Set up scales
- memcpy(aux, x[i].scales, 12);
- utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);
- utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);
- utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);
- utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);
-
- int8_t * scale = (int8_t *)utmp;
- for (int j = 0; j < 16; ++j) scale[j] -= m32;
-
- for (int j = 0; j < QK_K/128; ++j) {
-
- const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;
- const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;
- const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;
-
- q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);
- q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);
- q3h.val[2] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[0]), 1);
- q3h.val[3] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[1]), 1);
-
- q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[0], m3b)), vreinterpretq_s8_u8(q3h.val[0]));
- q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[1], m3b)), vreinterpretq_s8_u8(q3h.val[1]));
- q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));
- q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));
-
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];
-
- scale += 4;
-
- q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);
- q3h.val[1] = vbicq_u8(m2, qhbits.val[1]);
- q3h.val[2] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[0]), 1);
- q3h.val[3] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[1]), 1);
-
- q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 4), m3b)), vreinterpretq_s8_u8(q3h.val[0]));
- q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 4), m3b)), vreinterpretq_s8_u8(q3h.val[1]));
- q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));
- q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));
-
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];
-
- scale += 4;
-
- if (j == 0) {
- qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);
- qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);
- }
-
- }
- sum += d * isum;
-
- }
-
- *s = sum;
-
-#elif defined __AVX2__
-
- const __m256i m3 = _mm256_set1_epi8(3);
- const __m256i mone = _mm256_set1_epi8(1);
- const __m128i m32 = _mm_set1_epi8(32);
-
- __m256 acc = _mm256_setzero_ps();
-
- uint32_t aux[3];
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
-
- const uint8_t * restrict q3 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- // Set up scales
- memcpy(aux, x[i].scales, 12);
- __m128i scales128 = _mm_set_epi32(
- ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
- ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
- (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
- (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
- scales128 = _mm_sub_epi8(scales128, m32);
- const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);
- const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);
- const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);
- const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};
-
- // high bit
- const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);
-
- // integer accumulator
- __m256i sumi = _mm256_setzero_si256();
-
- int bit = 0;
- int is = 0;
-
- for (int j = 0; j < QK_K/128; ++j) {
- // load low 2 bits
- const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;
-
- // prepare low and high bits
- const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);
- const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
- ++bit;
-
- const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);
- const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
- ++bit;
-
- const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);
- const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
- ++bit;
-
- const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);
- const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);
- ++bit;
-
- // load Q8 quants
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
-
- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
- // and 2 if the high bit was set)
- __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);
- __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);
- __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);
- __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);
-
- __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);
- __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);
- __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);
- __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);
-
- p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
- p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
- p16_2 = _mm256_sub_epi16(p16_2, q8s_2);
- p16_3 = _mm256_sub_epi16(p16_3, q8s_3);
-
- // multiply with scales
- p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);
- p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);
- p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);
- p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);
-
- // accumulate
- p16_0 = _mm256_add_epi32(p16_0, p16_1);
- p16_2 = _mm256_add_epi32(p16_2, p16_3);
- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));
-
- }
-
- // multiply with block scale and accumulate
- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
-
- }
-
- *s = hsum_float_8(acc);
-
-#elif defined __AVX__
-
- const __m128i m3 = _mm_set1_epi8(3);
- const __m128i mone = _mm_set1_epi8(1);
- const __m128i m32 = _mm_set1_epi8(32);
- const __m128i m2 = _mm_set1_epi8(2);
-
- __m256 acc = _mm256_setzero_ps();
-
- const uint32_t *aux;
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
-
- const uint8_t * restrict q3 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- // Set up scales
- aux = (const uint32_t *)x[i].scales;
- __m128i scales128 = _mm_set_epi32(
- ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),
- ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),
- (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),
- (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));
- scales128 = _mm_sub_epi8(scales128, m32);
- const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);
- const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));
- const __m128i scales[2] = { scales_0, scales_1 };
-
- // high bit *128*2 from block_q3_K.hmask[QK_K/8]
- const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);
- const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);
-
- // integer accumulator
- __m128i sumi_0 = _mm_setzero_si128();
- __m128i sumi_1 = _mm_setzero_si128();
-
- for (int j = 0; j < QK_K/128; ++j) {
- // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]
- const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;
- const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;
-
- // prepare low and high bits
- const int bit = j << 2;
-
- const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);
- const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);
- const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);
- const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);
-
- const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);
- const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);
- const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);
- const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);
-
- const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);
- const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);
- const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);
- const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);
-
- const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);
- const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);
- const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);
- const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);
-
- // load Q8 quants from block_q8_K.qs[QK_K]
- const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
-
- // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,
- // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,
- // and 2 if the high bit was set)
- __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);
- __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);
- __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);
- __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);
- __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);
- __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);
- __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);
- __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);
-
- __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);
- __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);
- __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);
- __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);
- __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);
- __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);
- __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);
- __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);
-
- p16_0 = _mm_sub_epi16(p16_0, q8s_0);
- p16_1 = _mm_sub_epi16(p16_1, q8s_1);
- p16_2 = _mm_sub_epi16(p16_2, q8s_2);
- p16_3 = _mm_sub_epi16(p16_3, q8s_3);
- p16_4 = _mm_sub_epi16(p16_4, q8s_4);
- p16_5 = _mm_sub_epi16(p16_5, q8s_5);
- p16_6 = _mm_sub_epi16(p16_6, q8s_6);
- p16_7 = _mm_sub_epi16(p16_7, q8s_7);
-
- // multiply with scales
- __m128i shuffle = _mm_set1_epi16(0x0100);
- p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);
- shuffle = _mm_add_epi16(shuffle, m2);
- p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);
- shuffle = _mm_add_epi16(shuffle, m2);
- p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);
- shuffle = _mm_add_epi16(shuffle, m2);
- p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);
- shuffle = _mm_add_epi16(shuffle, m2);
- p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);
- shuffle = _mm_add_epi16(shuffle, m2);
- p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);
- shuffle = _mm_add_epi16(shuffle, m2);
- p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);
- shuffle = _mm_add_epi16(shuffle, m2);
- p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);
-
- // accumulate
- p16_0 = _mm_add_epi32(p16_0, p16_1);
- p16_2 = _mm_add_epi32(p16_2, p16_3);
- p16_4 = _mm_add_epi32(p16_4, p16_5);
- p16_6 = _mm_add_epi32(p16_6, p16_7);
- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));
-
- }
-
- // multiply with block scale and accumulate
- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);
-
- }
-
- *s = hsum_float_8(acc);
-
-#elif defined __riscv_v_intrinsic
-
- uint32_t aux[3];
- uint32_t utmp[4];
-
- float sumf = 0;
- for (int i = 0; i < nb; ++i) {
-
- const uint8_t * restrict q3 = x[i].qs;
- const uint8_t * restrict qh = x[i].hmask;
- const int8_t * restrict q8 = y[i].qs;
-
- memcpy(aux, x[i].scales, 12);
- utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);
- utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);
- utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);
- utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);
-
- int8_t * scale = (int8_t *)utmp;
- for (int j = 0; j < 16; ++j) scale[j] -= 32;
-
-
- size_t vl = 32;
- uint8_t m = 1;
-
- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
- vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl);
-
- int sum_t = 0;
-
- for (int j = 0; j < QK_K; j += 128) {
-
- vl = 32;
-
- // load Q3
- vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl);
-
- vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl));
- vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl));
- vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl));
- vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl));
-
- // compute mask for subtraction
- vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl);
- vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl);
- vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_m(vmask_0, q3_0, 0x4, vl);
- m <<= 1;
-
- vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);
- vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl);
- vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_m(vmask_1, q3_1, 0x4, vl);
- m <<= 1;
-
- vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);
- vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl);
- vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_m(vmask_2, q3_2, 0x4, vl);
- m <<= 1;
-
- vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl);
- vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl);
- vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_m(vmask_3, q3_3, 0x4, vl);
- m <<= 1;
-
- // load Q8 and take product with Q3
- vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl);
- vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl);
- vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl);
- vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl);
-
- vl = 16;
-
- // retrieve lane to multiply with scale
- vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl);
- vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl);
- vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl);
- vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl);
- vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl);
- vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl);
- vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl);
- vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl);
-
- vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl);
- vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl);
- vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl);
- vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl);
-
- sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);
-
- q3 += 32; q8 += 128; scale += 8;
-
- }
-
- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
-
- sumf += d*sum_t;
-
- }
-
- *s = sumf;
-
-#else
- // scalar version
- // This function is written like this so the compiler can manage to vectorize most of it
- // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the
- // manually vectorized version above. Every other version I tried would run at least 4 times slower.
- // The ideal situation would be if we could just write the code once, and the compiler would
- // automatically produce the best possible set of machine instructions, instead of us having to manually
- // write vectorized versions for AVX, ARM_NEON, etc.
-
- int8_t aux8[QK_K];
- int16_t aux16[8];
- float sums [8];
- int32_t aux32[8];
- memset(sums, 0, 8*sizeof(float));
-
- uint32_t auxs[4];
- const int8_t * scales = (const int8_t*)auxs;
-
- float sumf = 0;
- for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q3 = x[i].qs;
- const uint8_t * restrict hm = x[i].hmask;
- const int8_t * restrict q8 = y[i].qs;
- memset(aux32, 0, 8*sizeof(int32_t));
- int8_t * restrict a = aux8;
- uint8_t m = 1;
- for (int j = 0; j < QK_K; j += 128) {
- for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
- a += 32; m <<= 1;
- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
- a += 32; m <<= 1;
- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
- a += 32; m <<= 1;
- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
- a += 32; m <<= 1;
- q3 += 32;
- }
- a = aux8;
-
- memcpy(auxs, x[i].scales, 12);
- uint32_t tmp = auxs[2];
- auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
- auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
- auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
- auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
- for (int j = 0; j < QK_K/16; ++j) {
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
- q8 += 8; a += 8;
- }
- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
- }
- for (int l = 0; l < 8; ++l) sumf += sums[l];
- *s = sumf;
-
-#endif
-
-}
diff --git a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc
deleted file mode 100644
index b99edb44..00000000
--- a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc
+++ /dev/null
@@ -1,366 +0,0 @@
-static void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- const int qk = QK8_0;
- const int nb = n / qk;
-
- assert(n % qk == 0);
-#if defined(__ARM_FEATURE_MATMUL_INT8)
- assert((nrc == 2) || (nrc == 1));
-#else
- assert(nrc == 1);
-#endif
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
- const block_q4_0 * restrict x = vx;
- const block_q8_0 * restrict y = vy;
-
-#if defined(__ARM_FEATURE_MATMUL_INT8)
- if (nrc == 2) {
- const block_q4_0 * restrict vx0 = vx;
- const block_q4_0 * restrict vx1 = vx + bx;
-
- const block_q8_0 * restrict vy0 = vy;
- const block_q8_0 * restrict vy1 = vy + by;
-
- float32x4_t sumv0 = vdupq_n_f32(0.0f);
-
- for (int i = 0; i < nb; i++) {
- const block_q4_0 * restrict b_x0 = &vx0[i];
- const block_q4_0 * restrict b_x1 = &vx1[i];
- const block_q8_0 * restrict b_y0 = &vy0[i];
- const block_q8_0 * restrict b_y1 = &vy1[i];
-
- const uint8x16_t m4b = vdupq_n_u8(0x0F);
- const int8x16_t s8b = vdupq_n_s8(0x8);
-
- const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);
- const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);
-
- // 4-bit -> 8-bit
- const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
- const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
- const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
- const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
-
- // sub 8
- const int8x16_t x0_l = vsubq_s8(v0_0l, s8b);
- const int8x16_t x0_h = vsubq_s8(v0_0h, s8b);
- const int8x16_t x1_l = vsubq_s8(v0_1l, s8b);
- const int8x16_t x1_h = vsubq_s8(v0_1h, s8b);
-
- // load y
- const int8x16_t y0_l = vld1q_s8(b_y0->qs);
- const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
- const int8x16_t y1_l = vld1q_s8(b_y1->qs);
- const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
-
- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
- GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
-
- int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
- int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
-
- int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
- int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
-
- int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
- int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
-
- int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
- int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
-
- sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
- l1, r1)), l2, r2)), l3, r3))), scale);
- }
- float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
- float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
-
- vst1_f32(s, vget_low_f32(sumv2));
- vst1_f32(s + bs, vget_high_f32(sumv2));
- return;
- }
-#endif
-#if defined(__ARM_NEON)
- float32x4_t sumv0 = vdupq_n_f32(0.0f);
- float32x4_t sumv1 = vdupq_n_f32(0.0f);
-
- assert(nb % 2 == 0); // TODO: handle odd nb
-
- for (int i = 0; i < nb; i += 2) {
- const block_q4_0 * restrict x0 = &x[i + 0];
- const block_q4_0 * restrict x1 = &x[i + 1];
- const block_q8_0 * restrict y0 = &y[i + 0];
- const block_q8_0 * restrict y1 = &y[i + 1];
-
- const uint8x16_t m4b = vdupq_n_u8(0x0F);
- const int8x16_t s8b = vdupq_n_s8(0x8);
-
- const uint8x16_t v0_0 = vld1q_u8(x0->qs);
- const uint8x16_t v0_1 = vld1q_u8(x1->qs);
-
- // 4-bit -> 8-bit
- const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
- const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
- const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
- const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
-
- // sub 8
- const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);
- const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);
- const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);
- const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);
-
- // load y
- const int8x16_t v1_0l = vld1q_s8(y0->qs);
- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
- const int8x16_t v1_1l = vld1q_s8(y1->qs);
- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
-
- // dot product into int32x4_t
- const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);
- const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);
-
- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
- }
-
- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
-#elif defined(__AVX2__)
- // Initialize accumulator with zeros
- __m256 acc = _mm256_setzero_ps();
-
- // Main loop
- for (int i = 0; i < nb; ++i) {
- /* Compute combined scale for the block */
- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );
-
- __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);
-
- // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.
- const __m256i off = _mm256_set1_epi8( 8 );
- qx = _mm256_sub_epi8( qx, off );
-
- __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
-
- const __m256 q = mul_sum_i8_pairs_float_avx2(qx, qy);
-
- /* Multiply q with scale and accumulate */
- acc = _mm256_fmadd_ps( d, q, acc );
- }
-
- *s = hsum_float_8(acc);
-#elif defined(__AVX__)
- // Initialize accumulator with zeros
- __m256 acc = _mm256_setzero_ps();
-
- // Main loop
- for (int i = 0; i < nb; ++i) {
- // Compute combined scale for the block
- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );
-
- const __m128i lowMask = _mm_set1_epi8(0xF);
- const __m128i off = _mm_set1_epi8(8);
-
- const __m128i tmp = _mm_loadu_si128((const __m128i *)x[i].qs);
-
- __m128i bx_0 = _mm_and_si128(lowMask, tmp);
- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);
- bx_0 = _mm_sub_epi8(bx_0, off);
- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
-
- bx_0 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4));
- by_0 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));
- bx_0 = _mm_sub_epi8(bx_0, off);
- const __m128i i32_1 = mul_sum_i8_pairs(bx_0, by_0);
-
- // Convert int32_t to float
- __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1));
-
- // Apply the scale, and accumulate
- acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);
- }
-
- *s = hsum_float_8(acc);
-#elif defined(__SSSE3__)
- // set constants
- const __m128i lowMask = _mm_set1_epi8(0xF);
- const __m128i off = _mm_set1_epi8(8);
-
- // Initialize accumulator with zeros
- __m128 acc_0 = _mm_setzero_ps();
- __m128 acc_1 = _mm_setzero_ps();
- __m128 acc_2 = _mm_setzero_ps();
- __m128 acc_3 = _mm_setzero_ps();
-
- // First round without accumulation
- {
- _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0);
- _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0);
-
- // Compute combined scale for the block 0 and 1
- const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) );
-
- const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[0].qs);
-
- __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);
- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[0].qs);
- bx_0 = _mm_sub_epi8(bx_0, off);
- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
-
- __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));
- __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[0].qs + 16));
- bx_1 = _mm_sub_epi8(bx_1, off);
- const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
-
- _mm_prefetch(&x[1] + sizeof(block_q4_0), _MM_HINT_T0);
- _mm_prefetch(&y[1] + sizeof(block_q8_0), _MM_HINT_T0);
-
- // Compute combined scale for the block 2 and 3
- const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) );
-
- const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[1].qs);
-
- __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);
- __m128i by_2 = _mm_loadu_si128((const __m128i *)y[1].qs);
- bx_2 = _mm_sub_epi8(bx_2, off);
- const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);
-
- __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));
- __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[1].qs + 16));
- bx_3 = _mm_sub_epi8(bx_3, off);
- const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);
-
- // Convert int32_t to float
- __m128 p0 = _mm_cvtepi32_ps(i32_0);
- __m128 p1 = _mm_cvtepi32_ps(i32_1);
- __m128 p2 = _mm_cvtepi32_ps(i32_2);
- __m128 p3 = _mm_cvtepi32_ps(i32_3);
-
- // Apply the scale
- acc_0 = _mm_mul_ps( d_0_1, p0 );
- acc_1 = _mm_mul_ps( d_0_1, p1 );
- acc_2 = _mm_mul_ps( d_2_3, p2 );
- acc_3 = _mm_mul_ps( d_2_3, p3 );
- }
-
- assert(nb % 2 == 0); // TODO: handle odd nb
-
- // Main loop
- for (int i = 2; i < nb; i+=2) {
- _mm_prefetch(&x[i] + sizeof(block_q4_0), _MM_HINT_T0);
- _mm_prefetch(&y[i] + sizeof(block_q8_0), _MM_HINT_T0);
-
- // Compute combined scale for the block 0 and 1
- const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );
-
- const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[i].qs);
-
- __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);
- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);
- bx_0 = _mm_sub_epi8(bx_0, off);
- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);
-
- __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));
- __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));
- bx_1 = _mm_sub_epi8(bx_1, off);
- const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
-
- _mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
- _mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
-
- // Compute combined scale for the block 2 and 3
- const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) );
-
- const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[i + 1].qs);
-
- __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);
- __m128i by_2 = _mm_loadu_si128((const __m128i *)y[i + 1].qs);
- bx_2 = _mm_sub_epi8(bx_2, off);
- const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);
-
- __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));
- __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[i + 1].qs + 16));
- bx_3 = _mm_sub_epi8(bx_3, off);
- const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);
-
- // Convert int32_t to float
- __m128 p0 = _mm_cvtepi32_ps(i32_0);
- __m128 p1 = _mm_cvtepi32_ps(i32_1);
- __m128 p2 = _mm_cvtepi32_ps(i32_2);
- __m128 p3 = _mm_cvtepi32_ps(i32_3);
-
- // Apply the scale
- __m128 p0_d = _mm_mul_ps( d_0_1, p0 );
- __m128 p1_d = _mm_mul_ps( d_0_1, p1 );
- __m128 p2_d = _mm_mul_ps( d_2_3, p2 );
- __m128 p3_d = _mm_mul_ps( d_2_3, p3 );
-
- // Acummulate
- acc_0 = _mm_add_ps(p0_d, acc_0);
- acc_1 = _mm_add_ps(p1_d, acc_1);
- acc_2 = _mm_add_ps(p2_d, acc_2);
- acc_3 = _mm_add_ps(p3_d, acc_3);
- }
-
- *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);
-#elif defined(__riscv_v_intrinsic)
- float sumf = 0.0;
-
- size_t vl = __riscv_vsetvl_e8m1(qk/2);
-
- for (int i = 0; i < nb; i++) {
- // load elements
- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);
-
- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);
- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);
-
- // mask and store lower part of x, and then upper part
- vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
- vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
-
- vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
- vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
-
- // subtract offset
- vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 8, vl);
- vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 8, vl);
-
- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
-
- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
-
- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
-
- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
-
- sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);
- }
-
- *s = sumf;
-#else
- // scalar
- float sumf = 0.0;
-
- for (int i = 0; i < nb; i++) {
- int sumi = 0;
-
- for (int j = 0; j < qk/2; ++j) {
- const int v0 = (x[i].qs[j] & 0x0F) - 8;
- const int v1 = (x[i].qs[j] >> 4) - 8;
-
- sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);
- }
-
- sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);
- }
-
- *s = sumf;
-#endif
-}
diff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc
deleted file mode 100644
index aeb4c8b9..00000000
--- a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc
+++ /dev/null
@@ -1,360 +0,0 @@
-static void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- assert(n % QK_K == 0);
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
- const block_q4_K * restrict x = vx;
- const block_q8_K * restrict y = vy;
-
- const int nb = n / QK_K;
-
- static const uint32_t kmask1 = 0x3f3f3f3f;
- static const uint32_t kmask2 = 0x0f0f0f0f;
- static const uint32_t kmask3 = 0x03030303;
-
- uint32_t utmp[4];
-
-#ifdef __ARM_NEON
- const uint8x16_t m4b = vdupq_n_u8(0xf);
- const int32x4_t mzero = vdupq_n_s32(0);
-
- ggml_int8x16x2_t q4bytes;
- ggml_int8x16x2_t q8bytes;
-
- float sumf = 0;
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));
-
- memcpy(utmp, x[i].scales, 12);
-
- uint32x2_t mins8 = { 0 };
- mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);
- mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);
-
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[0] &= kmask1;
-
- const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));
- const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),
- vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));
- sumf -= dmin * vaddvq_s32(prod);
-
- const uint8_t * scales = (const uint8_t *)utmp;
-
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- int32_t sumi1 = 0;
- int32_t sumi2 = 0;
-
- for (int j = 0; j < QK_K/64; ++j) {
- const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;
-
- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));
- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));
-
- const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
- sumi1 += vaddvq_s32(p1) * scales[2*j+0];
-
- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;
- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));
- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));
-
- const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);
-
- sumi2 += vaddvq_s32(p2) * scales[2*j+1];
- }
-
- sumf += d * (sumi1 + sumi2);
-
- }
-
- *s = sumf;
-
-#elif defined __AVX2__
-
- const __m256i m4 = _mm256_set1_epi8(0xF);
-
- __m256 acc = _mm256_setzero_ps();
- __m128 acc_m = _mm_setzero_ps();
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
-
- const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
- const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
- const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
- acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);
-
- const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);
- const __m256i scales = MM256_SET_M128I(sc128, sc128);
-
- __m256i sumi = _mm256_setzero_si256();
-
- for (int j = 0; j < QK_K/64; ++j) {
-
- const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));
- const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));
-
- const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
- const __m256i q4l = _mm256_and_si256(q4bits, m4);
- const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);
-
- const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);
- p16l = _mm256_madd_epi16(scale_l, p16l);
-
- const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);
- p16h = _mm256_madd_epi16(scale_h, p16h);
- const __m256i sumj = _mm256_add_epi32(p16l, p16h);
-
- sumi = _mm256_add_epi32(sumi, sumj);
- }
-
- __m256 vd = _mm256_set1_ps(d);
- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);
-
- }
-
- acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
- acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
-
- *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);
-
-#elif defined __AVX__
-
- const __m128i m4 = _mm_set1_epi8(0xF);
- const __m128i m2 = _mm_set1_epi8(0x2);
-
- __m256 acc = _mm256_setzero_ps();
- __m128 acc_m = _mm_setzero_ps();
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);
- const __m128i scales = _mm_cvtepu8_epi16(utmps);
- const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));
-
- const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);
- const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);
- const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);
- const __m128i prod = _mm_madd_epi16(mins, q8s);
- acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);
-
- __m128i sumi_0 = _mm_setzero_si128();
- __m128i sumi_1 = _mm_setzero_si128();
-
- __m128i shuffle = _mm_set1_epi16(0x0100);
- for (int j = 0; j < QK_K/64; ++j) {
-
- const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi16(shuffle, m2);
- const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi16(shuffle, m2);
-
- __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
- const __m128i q4l_0 = _mm_and_si128(q4bits, m4);
- const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);
- q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
- const __m128i q4l_1 = _mm_and_si128(q4bits, m4);
- const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);
-
- const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);
- p16l = _mm_madd_epi16(scale_l, p16l);
- sumi_0 = _mm_add_epi32(sumi_0, p16l);
- const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- p16l = _mm_maddubs_epi16(q4l_1, q8l_1);
- p16l = _mm_madd_epi16(scale_l, p16l);
- sumi_1 = _mm_add_epi32(sumi_1, p16l);
-
- const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);
- p16h = _mm_madd_epi16(scale_h, p16h);
- sumi_0 = _mm_add_epi32(sumi_0, p16h);
- const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- p16h = _mm_maddubs_epi16(q4h_1, q8h_1);
- p16h = _mm_madd_epi16(scale_h, p16h);
- sumi_1 = _mm_add_epi32(sumi_1, p16h);
-
- }
-
- __m256 vd = _mm256_set1_ps(d);
- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);
-
- }
-
- acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
- acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
-
- *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);
-
-#elif defined __riscv_v_intrinsic
-
- const uint8_t * scales = (const uint8_t*)&utmp[0];
- const uint8_t * mins = (const uint8_t*)&utmp[2];
-
- float sumf = 0;
-
- for (int i = 0; i < nb; ++i) {
-
- size_t vl = 8;
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);
- vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);
- vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);
-
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);
- vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));
- vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);
-
- vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
- sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);
-
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- vl = 32;
-
- int32_t sum_1 = 0;
- int32_t sum_2 = 0;
-
- vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);
-
- for (int j = 0; j < QK_K/64; ++j) {
- // load Q4
- vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);
-
- // load Q8 and multiply it with lower Q4 nibble
- vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);
- vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));
- vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);
- vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);
-
- sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];
-
- // load Q8 and multiply it with upper Q4 nibble
- vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);
- vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));
- vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);
- vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);
-
- sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];
-
- q4 += 32; q8 += 64;
-
- }
-
- sumf += d*(sum_1 + sum_2);
-
- }
-
- *s = sumf;
-
-#else
-
-
- const uint8_t * scales = (const uint8_t*)&utmp[0];
- const uint8_t * mins = (const uint8_t*)&utmp[2];
-
- int8_t aux8[QK_K];
- int16_t aux16[8];
- float sums [8];
- int32_t aux32[8];
- memset(sums, 0, 8*sizeof(float));
-
- float sumf = 0;
- for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q4 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
- memset(aux32, 0, 8*sizeof(int32_t));
- int8_t * restrict a = aux8;
- for (int j = 0; j < QK_K/64; ++j) {
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
- a += 32;
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
- a += 32; q4 += 32;
- }
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- int sumi = 0;
- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
- a = aux8;
- int is = 0;
- for (int j = 0; j < QK_K/32; ++j) {
- int32_t scale = scales[is++];
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- }
- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
- const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
- sumf -= dmin * sumi;
- }
- for (int l = 0; l < 8; ++l) sumf += sums[l];
- *s = sumf;
-#endif
-}
diff --git a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc
deleted file mode 100644
index 8ced89f2..00000000
--- a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc
+++ /dev/null
@@ -1,290 +0,0 @@
-static void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- const int qk = QK8_0;
- const int nb = n / qk;
-
- assert(n % qk == 0);
- assert(qk == QK5_0);
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
- const block_q5_0 * restrict x = vx;
- const block_q8_0 * restrict y = vy;
-
-#if defined(__ARM_NEON)
- float32x4_t sumv0 = vdupq_n_f32(0.0f);
- float32x4_t sumv1 = vdupq_n_f32(0.0f);
-
- uint32_t qh0;
- uint32_t qh1;
-
- uint64_t tmp0[4];
- uint64_t tmp1[4];
-
- assert(nb % 2 == 0); // TODO: handle odd nb
-
- for (int i = 0; i < nb; i += 2) {
- const block_q5_0 * restrict x0 = &x[i];
- const block_q5_0 * restrict x1 = &x[i + 1];
- const block_q8_0 * restrict y0 = &y[i];
- const block_q8_0 * restrict y1 = &y[i + 1];
-
- const uint8x16_t m4b = vdupq_n_u8(0x0F);
-
- // extract the 5th bit via lookup table ((!b) << 4)
- memcpy(&qh0, x0->qh, sizeof(qh0));
- memcpy(&qh1, x1->qh, sizeof(qh1));
-
- tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];
- tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];
- tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];
- tmp0[3] = table_b2b_1[(qh0 >> 24) ];
-
- tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];
- tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];
- tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];
- tmp1[3] = table_b2b_1[(qh1 >> 24) ];
-
- const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));
- const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));
- const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));
- const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));
-
- const uint8x16_t v0_0 = vld1q_u8(x0->qs);
- const uint8x16_t v0_1 = vld1q_u8(x1->qs);
-
- // 4-bit -> 8-bit
- int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
- int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
- int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
- int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
-
- // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
- const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);
- const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);
- const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);
- const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);
-
- // load y
- const int8x16_t v1_0l = vld1q_s8(y0->qs);
- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);
- const int8x16_t v1_1l = vld1q_s8(y1->qs);
- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);
-
- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
- ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),
- ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
- ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),
- ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
- }
-
- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
-#elif defined(__wasm_simd128__)
- v128_t sumv = wasm_f32x4_splat(0.0f);
-
- uint32_t qh;
- uint64_t tmp[4];
-
- // TODO: check if unrolling this is better
- for (int i = 0; i < nb; ++i) {
- const block_q5_0 * restrict x0 = &x[i];
- const block_q8_0 * restrict y0 = &y[i];
-
- const v128_t m4b = wasm_i8x16_splat(0x0F);
-
- // extract the 5th bit
- memcpy(&qh, x0->qh, sizeof(qh));
-
- tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];
- tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];
- tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];
- tmp[3] = table_b2b_1[(qh >> 24) ];
-
- const v128_t qhl = wasm_v128_load(tmp + 0);
- const v128_t qhh = wasm_v128_load(tmp + 2);
-
- const v128_t v0 = wasm_v128_load(x0->qs);
-
- // 4-bit -> 8-bit
- const v128_t v0l = wasm_v128_and (v0, m4b);
- const v128_t v0h = wasm_u8x16_shr(v0, 4);
-
- // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
- const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
- const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
-
- // load y
- const v128_t v1l = wasm_v128_load(y0->qs);
- const v128_t v1h = wasm_v128_load(y0->qs + 16);
-
- // int8x16 -> int16x8
- const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
- const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
- const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);
- const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);
-
- const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);
- const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);
- const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);
- const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);
-
- // dot product
- sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(
- wasm_i32x4_add(
- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),
- wasm_i32x4_dot_i16x8(v0lfh, v1lh)),
- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
- wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
- wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));
- }
-
- *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
- wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);
-#elif defined(__AVX2__)
- // Initialize accumulator with zeros
- __m256 acc = _mm256_setzero_ps();
-
- // Main loop
- for (int i = 0; i < nb; i++) {
- /* Compute combined scale for the block */
- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
-
- __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);
- __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);
- bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));
- qx = _mm256_or_si256(qx, bxhi);
-
- __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
-
- const __m256 q = mul_sum_i8_pairs_float_avx2(qx, qy);
-
- /* Multiply q with scale and accumulate */
- acc = _mm256_fmadd_ps(d, q, acc);
- }
-
- *s = hsum_float_8(acc);
-#elif defined(__AVX__)
- // Initialize accumulator with zeros
- __m256 acc = _mm256_setzero_ps();
- __m128i mask = _mm_set1_epi8((char)0xF0);
-
- // Main loop
- for (int i = 0; i < nb; i++) {
- /* Compute combined scale for the block */
- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
-
- __m256i bx_0 = bytes_from_nibbles_32_avx(x[i].qs);
- const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);
- __m128i bxhil = _mm256_castsi256_si128(bxhi);
- __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);
- bxhil = _mm_andnot_si128(bxhil, mask);
- bxhih = _mm_andnot_si128(bxhih, mask);
- __m128i bxl = _mm256_castsi256_si128(bx_0);
- __m128i bxh = _mm256_extractf128_si256(bx_0, 1);
- bxl = _mm_or_si128(bxl, bxhil);
- bxh = _mm_or_si128(bxh, bxhih);
- bx_0 = MM256_SET_M128I(bxh, bxl);
-
- const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs);
-
- const __m256 q = mul_sum_i8_pairs_float_avx(bx_0, by_0);
-
- /* Multiply q with scale and accumulate */
- acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);
- }
-
- *s = hsum_float_8(acc);
-#elif defined(__riscv_v_intrinsic)
- float sumf = 0.0;
-
- uint32_t qh;
-
- size_t vl = __riscv_vsetvl_e8m1(qk/2);
-
- // These temporary registers are for masking and shift operations
- vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);
- vuint32m2_t vt_2 = __riscv_vsll_vv_u32m2(__riscv_vmv_v_x_u32m2(1, vl), vt_1, vl);
-
- vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);
- vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);
-
- for (int i = 0; i < nb; i++) {
- memcpy(&qh, x[i].qh, sizeof(uint32_t));
-
- // ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
- vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);
- vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(xha_0, vt_1, vl);
- vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);
-
- // ((qh & (1u << (j + 16))) >> (j + 12));
- vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(vt_3, qh, vl);
- vuint32m2_t xhl_1 = __riscv_vsrl_vv_u32m2(xha_1, vt_4, vl);
-
- // narrowing
- vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xhl_0, vl);
- vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);
-
- vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xhl_1, vl);
- vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);
-
- // load
- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);
-
- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);
- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);
-
- vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
- vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
-
- vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);
- vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);
-
- vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);
- vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);
-
- vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 16, vl);
- vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 16, vl);
-
- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);
- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);
-
- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);
-
- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);
- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);
-
- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
-
- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;
- }
-
- *s = sumf;
-#else
- // scalar
- float sumf = 0.0;
-
- for (int i = 0; i < nb; i++) {
- uint32_t qh;
- memcpy(&qh, x[i].qh, sizeof(qh));
-
- int sumi = 0;
-
- for (int j = 0; j < qk/2; ++j) {
- const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
- const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));
-
- const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;
- const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;
-
- sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);
- }
-
- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;
- }
-
- *s = sumf;
-#endif
-}
diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc
deleted file mode 100644
index 2393e0d2..00000000
--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc
+++ /dev/null
@@ -1,422 +0,0 @@
-static void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- assert(n % QK_K == 0);
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
- const block_q5_K * restrict x = vx;
- const block_q8_K * restrict y = vy;
-
- const int nb = n / QK_K;
-
- static const uint32_t kmask1 = 0x3f3f3f3f;
- static const uint32_t kmask2 = 0x0f0f0f0f;
- static const uint32_t kmask3 = 0x03030303;
-
- uint32_t utmp[4];
-
-#ifdef __ARM_NEON
- const uint8x16_t m4b = vdupq_n_u8(0xf);
- const uint8x16_t mone = vdupq_n_u8(1);
- const uint8x16_t mtwo = vdupq_n_u8(2);
- const int32x4_t mzero = vdupq_n_s32(0);
-
- ggml_int8x16x4_t q5bytes;
-
- float sumf = 0;
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));
-
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- const uint8x8_t mins8 = vld1_u8((const uint8_t*)utmp + 8);
- const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(mins8));
- const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),
- vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));
- int32_t sumi_mins = vaddvq_s32(prod);
-
- const uint8_t * scales = (const uint8_t *)utmp;
-
- const uint8_t * restrict q5 = x[i].qs;
- const uint8_t * restrict qh = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
-
- ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);
-
- ggml_uint8x16x4_t q5h;
-
- int32_t sumi = 0;
-
- for (int j = 0; j < QK_K/64; ++j) {
-
- const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;
- const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
-
- q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);
- q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);
- q5h.val[2] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[0]), 3);
- q5h.val[3] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[1]), 3);
- qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 2);
- qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 2);
-
- q5bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[0], m4b), q5h.val[0]));
- q5bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[1], m4b), q5h.val[1]));
- q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));
- q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));
-
- sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;
- sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;
- }
-
- sumf += d * sumi - dmin * sumi_mins;
- }
-
- *s = sumf;
-
-#elif defined __AVX2__
-
- const __m256i m4 = _mm256_set1_epi8(0xF);
- const __m128i mzero = _mm_setzero_si128();
- const __m256i mone = _mm256_set1_epi8(1);
-
- __m256 acc = _mm256_setzero_ps();
-
- float summs = 0.f;
-
- for (int i = 0; i < nb; ++i) {
-
- const uint8_t * restrict q5 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
-#if QK_K == 256
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-#else
- // TODO
- const float d = 0, dmin = 0;
-#endif
-
- const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
-
- const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
- const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
- const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
- const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);
- summs += dmin * _mm_extract_epi32(hsum, 0);
-
- const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);
- const __m256i scales = MM256_SET_M128I(sc128, sc128);
-
- const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);
- __m256i hmask = mone;
-
- __m256i sumi = _mm256_setzero_si256();
-
- int bit = 0;
-
- for (int j = 0; j < QK_K/64; ++j) {
-
- const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));
- const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));
-
- const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;
-
- const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);
- const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);
- const __m256i q5_0 = _mm256_add_epi8(q5l_0, q5h_0);
- hmask = _mm256_slli_epi16(hmask, 1);
-
- const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);
- const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);
- const __m256i q5_1 = _mm256_add_epi8(q5l_1, q5h_1);
- hmask = _mm256_slli_epi16(hmask, 1);
-
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
-
- __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);
- __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);
-
- p16_0 = _mm256_madd_epi16(scale_0, p16_0);
- p16_1 = _mm256_madd_epi16(scale_1, p16_1);
-
- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));
-
- }
-
- __m256 vd = _mm256_set1_ps(d);
- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);
-
- }
-
- *s = hsum_float_8(acc) + summs;
-
-#elif defined __AVX__
-
- const __m128i m4 = _mm_set1_epi8(0xF);
- const __m128i mzero = _mm_setzero_si128();
- const __m128i mone = _mm_set1_epi8(1);
- const __m128i m2 = _mm_set1_epi8(2);
-
- __m256 acc = _mm256_setzero_ps();
-
- float summs = 0.f;
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
-
- const uint8_t * restrict q5 = x[i].qs;
- const int8_t * restrict q8 = y[i].qs;
-
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);
- const __m128i scales = _mm_cvtepu8_epi16(utmps);
- const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));
-
- const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);
- const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);
- const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);
- const __m128i prod = _mm_madd_epi16(mins, q8s);
- const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);
- summs += dmin * _mm_extract_epi32(hsum, 0);
-
- const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);
- const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);
- __m128i hmask = mone;
-
- __m128i sumi_0 = _mm_setzero_si128();
- __m128i sumi_1 = _mm_setzero_si128();
-
- int bit = 0;
-
- __m128i shuffle = _mm_set1_epi16(0x0100);
- for (int j = 0; j < QK_K/64; ++j) {
-
- const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi16(shuffle, m2);
- const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi16(shuffle, m2);
-
- const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;
- const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;
-
- __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);
- __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);
- __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);
- __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);
- __m128i q5_0 = _mm_add_epi8(q5l_0, q5h_0);
- __m128i q5_1 = _mm_add_epi8(q5l_1, q5h_1);
- hmask = _mm_slli_epi16(hmask, 1);
-
- __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);
- __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);
- p16_0 = _mm_madd_epi16(scale_0, p16_0);
- p16_1 = _mm_madd_epi16(scale_0, p16_1);
-
- q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);
- q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);
- q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);
- q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);
- q5_0 = _mm_add_epi8(q5l_0, q5h_0);
- q5_1 = _mm_add_epi8(q5l_1, q5h_1);
- hmask = _mm_slli_epi16(hmask, 1);
-
- q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);
- __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);
- p16_2 = _mm_madd_epi16(scale_1, p16_2);
- p16_3 = _mm_madd_epi16(scale_1, p16_3);
-
- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));
-
- }
-
- __m256 vd = _mm256_set1_ps(d);
- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);
-
- }
-
- *s = hsum_float_8(acc) + summs;
-
-#elif defined __riscv_v_intrinsic
-
- const uint8_t * scales = (const uint8_t*)&utmp[0];
- const uint8_t * mins = (const uint8_t*)&utmp[2];
-
- float sumf = 0;
- float sums = 0.0;
-
- size_t vl;
-
- for (int i = 0; i < nb; ++i) {
-
- vl = 8;
-
- const uint8_t * restrict q5 = x[i].qs;
- const uint8_t * restrict hm = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
-
- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
- const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
-
- vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);
- vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);
- vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);
-
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);
- vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));
- vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);
-
- vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);
- sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);
-
- vl = 32;
- int32_t aux32 = 0;
- int is = 0;
-
- uint8_t m = 1;
- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
- vuint8m1_t vqh = __riscv_vle8_v_u8m1(hm, vl);
-
- for (int j = 0; j < QK_K/64; ++j) {
- // load Q5 and Q8
- vuint8m1_t q5_x = __riscv_vle8_v_u8m1(q5, vl);
- vint8m1_t q8_y1 = __riscv_vle8_v_i8m1(q8, vl);
- vint8m1_t q8_y2 = __riscv_vle8_v_i8m1(q8+32, vl);
-
- // compute mask for addition
- vint8m1_t q5_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q5_x, 0x0F, vl));
- vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);
- vbool8_t vmask_1 = __riscv_vmsne_vx_u8m1_b8(qh_m1, 0, vl);
- vint8m1_t q5_m1 = __riscv_vadd_vx_i8m1_m(vmask_1, q5_a, 16, vl);
- m <<= 1;
-
- vint8m1_t q5_l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q5_x, 0x04, vl));
- vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);
- vbool8_t vmask_2 = __riscv_vmsne_vx_u8m1_b8(qh_m2, 0, vl);
- vint8m1_t q5_m2 = __riscv_vadd_vx_i8m1_m(vmask_2, q5_l, 16, vl);
- m <<= 1;
-
- vint16m2_t v0 = __riscv_vwmul_vv_i16m2(q5_m1, q8_y1, vl);
- vint16m2_t v1 = __riscv_vwmul_vv_i16m2(q5_m2, q8_y2, vl);
-
- vint32m4_t vs1 = __riscv_vwmul_vx_i32m4(v0, scales[is++], vl);
- vint32m4_t vs2 = __riscv_vwmul_vx_i32m4(v1, scales[is++], vl);
-
- vint32m1_t vacc1 = __riscv_vredsum_vs_i32m4_i32m1(vs1, vzero, vl);
- vint32m1_t vacc2 = __riscv_vredsum_vs_i32m4_i32m1(vs2, vzero, vl);
-
- aux32 += __riscv_vmv_x_s_i32m1_i32(vacc1) + __riscv_vmv_x_s_i32m1_i32(vacc2);
- q5 += 32; q8 += 64;
-
- }
-
- vfloat32m1_t vaux = __riscv_vfmul_vf_f32m1(__riscv_vfmv_v_f_f32m1(aux32, 1), d, 1);
- sums += __riscv_vfmv_f_s_f32m1_f32(vaux);
-
- }
-
- *s = sumf+sums;
-
-#else
-
- const uint8_t * scales = (const uint8_t*)&utmp[0];
- const uint8_t * mins = (const uint8_t*)&utmp[2];
-
- int8_t aux8[QK_K];
- int16_t aux16[8];
- float sums [8];
- int32_t aux32[8];
- memset(sums, 0, 8*sizeof(float));
-
- float sumf = 0;
- for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q4 = x[i].qs;
- const uint8_t * restrict hm = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
- memset(aux32, 0, 8*sizeof(int32_t));
- int8_t * restrict a = aux8;
- uint8_t m = 1;
- for (int j = 0; j < QK_K/64; ++j) {
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
- a += 32; m <<= 1;
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
- a += 32; m <<= 1;
- q4 += 32;
- }
- memcpy(utmp, x[i].scales, 12);
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
- const uint32_t uaux = utmp[1] & kmask1;
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
- utmp[2] = uaux;
- utmp[0] &= kmask1;
-
- int sumi = 0;
- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
- a = aux8;
- int is = 0;
- for (int j = 0; j < QK_K/32; ++j) {
- int32_t scale = scales[is++];
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- }
- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
- const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;
- sumf -= dmin * sumi;
- }
- for (int l = 0; l < 8; ++l) sumf += sums[l];
- *s = sumf;
-#endif
-}
diff --git a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc
deleted file mode 100644
index 343d633d..00000000
--- a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc
+++ /dev/null
@@ -1,434 +0,0 @@
-static void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- assert(n % QK_K == 0);
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
- const block_q6_K * restrict x = vx;
- const block_q8_K * restrict y = vy;
-
- const int nb = n / QK_K;
-
-#ifdef __ARM_NEON
- float sum = 0;
-
- const uint8x16_t m4b = vdupq_n_u8(0xF);
- const int32x4_t vzero = vdupq_n_s32(0);
- //const int8x16_t m32s = vdupq_n_s8(32);
-
- const uint8x16_t mone = vdupq_n_u8(3);
-
- ggml_int8x16x4_t q6bytes;
- ggml_uint8x16x4_t q6h;
-
- for (int i = 0; i < nb; ++i) {
-
- const float d_all = GGML_FP16_TO_FP32(x[i].d);
-
- const uint8_t * restrict q6 = x[i].ql;
- const uint8_t * restrict qh = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
-
- const int8_t * restrict scale = x[i].scales;
-
- const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);
- const int8x16_t scales = vld1q_s8(scale);
- const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};
-
- const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),
- vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),
- vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),
- vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));
- int32_t isum_mins = vaddvq_s32(prod);
-
- int32_t isum = 0;
-
- for (int j = 0; j < QK_K/128; ++j) {
-
- ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;
- ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;
- ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
-
- q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);
- q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);
- uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);
- q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
- shifted = vshrq_n_u8(qhbits.val[1], 2);
- q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
-
- //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);
- //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);
- //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);
- //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);
- q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));
- q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));
- q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));
- q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));
-
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +
- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +
- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +
- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];
-
- scale += 4;
-
- q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;
-
- shifted = vshrq_n_u8(qhbits.val[0], 4);
- q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
- shifted = vshrq_n_u8(qhbits.val[1], 4);
- q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
- shifted = vshrq_n_u8(qhbits.val[0], 6);
- q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
- shifted = vshrq_n_u8(qhbits.val[1], 6);
- q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);
-
- //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);
- //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);
- //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);
- //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);
- q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));
- q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));
- q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));
- q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));
-
- isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +
- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +
- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +
- vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];
- scale += 4;
- }
- //sum += isum * d_all * y[i].d;
- sum += d_all * y[i].d * (isum - 32 * isum_mins);
-
- }
- *s = sum;
-
-#elif defined __AVX2__
-
- const __m256i m4 = _mm256_set1_epi8(0xF);
- const __m256i m2 = _mm256_set1_epi8(3);
- const __m256i m32s = _mm256_set1_epi8(32);
-
- __m256 acc = _mm256_setzero_ps();
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
-
- const uint8_t * restrict q4 = x[i].ql;
- const uint8_t * restrict qh = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
-
- const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);
-
- __m256i sumi = _mm256_setzero_si256();
-
- int is = 0;
-
- for (int j = 0; j < QK_K/128; ++j) {
-
- const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));
- const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));
- const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));
- const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));
- is += 4;
-
- const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
- const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;
- const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;
-
- const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);
- const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);
- const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);
- const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);
-
- const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);
- const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);
- const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);
- const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);
-
- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
- const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;
-
- __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);
- __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);
- __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);
- __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);
-
- __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);
- __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);
- __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);
- __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);
-
- p16_0 = _mm256_sub_epi16(p16_0, q8s_0);
- p16_1 = _mm256_sub_epi16(p16_1, q8s_1);
- p16_2 = _mm256_sub_epi16(p16_2, q8s_2);
- p16_3 = _mm256_sub_epi16(p16_3, q8s_3);
-
- p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);
- p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);
- p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);
- p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);
-
- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));
- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));
-
- }
-
- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);
- }
-
- *s = hsum_float_8(acc);
-
-#elif defined __AVX__
-
- const __m128i m4 = _mm_set1_epi8(0xF);
- const __m128i m3 = _mm_set1_epi8(3);
- const __m128i m32s = _mm_set1_epi8(32);
- const __m128i m2 = _mm_set1_epi8(2);
-
- __m256 acc = _mm256_setzero_ps();
-
- for (int i = 0; i < nb; ++i) {
-
- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
-
- const uint8_t * restrict q4 = x[i].ql;
- const uint8_t * restrict qh = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
-
- const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);
-
- __m128i sumi_0 = _mm_setzero_si128();
- __m128i sumi_1 = _mm_setzero_si128();
-
- __m128i shuffle = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
- for (int j = 0; j < QK_K/128; ++j) {
-
- const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;
- const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;
-
- const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);
- const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);
- const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 2), m3), 4);
- const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 2), m3), 4);
- const __m128i q4h_4 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 4), m3), 4);
- const __m128i q4h_5 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 4), m3), 4);
- const __m128i q4h_6 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 6), m3), 4);
- const __m128i q4h_7 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 6), m3), 4);
-
- const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
- const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
- const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
- const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;
-
- const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m4), q4h_0);
- const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m4), q4h_1);
- const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m4), q4h_2);
- const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m4), q4h_3);
- const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m4), q4h_4);
- const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m4), q4h_5);
- const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m4), q4h_6);
- const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m4), q4h_7);
-
- const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
- const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;
-
- __m128i q8s_0 = _mm_maddubs_epi16(m32s, q8_0);
- __m128i q8s_1 = _mm_maddubs_epi16(m32s, q8_1);
- __m128i q8s_2 = _mm_maddubs_epi16(m32s, q8_2);
- __m128i q8s_3 = _mm_maddubs_epi16(m32s, q8_3);
- __m128i q8s_4 = _mm_maddubs_epi16(m32s, q8_4);
- __m128i q8s_5 = _mm_maddubs_epi16(m32s, q8_5);
- __m128i q8s_6 = _mm_maddubs_epi16(m32s, q8_6);
- __m128i q8s_7 = _mm_maddubs_epi16(m32s, q8_7);
-
- __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);
- __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);
- __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);
- __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);
- __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);
- __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);
- __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);
- __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);
-
- p16_0 = _mm_sub_epi16(p16_0, q8s_0);
- p16_1 = _mm_sub_epi16(p16_1, q8s_1);
- p16_2 = _mm_sub_epi16(p16_2, q8s_2);
- p16_3 = _mm_sub_epi16(p16_3, q8s_3);
- p16_4 = _mm_sub_epi16(p16_4, q8s_4);
- p16_5 = _mm_sub_epi16(p16_5, q8s_5);
- p16_6 = _mm_sub_epi16(p16_6, q8s_6);
- p16_7 = _mm_sub_epi16(p16_7, q8s_7);
-
- const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi8(shuffle, m2);
- const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi8(shuffle, m2);
- const __m128i scale_2 = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi8(shuffle, m2);
- const __m128i scale_3 = _mm_shuffle_epi8(scales, shuffle);
- shuffle = _mm_add_epi8(shuffle, m2);
-
- p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);
- p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1);
- p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);
- p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3);
- p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);
- p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_2, scale_2)), p16_5);
- p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);
- p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_3, scale_3)), p16_7);
-
- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));
- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));
- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));
- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));
-
- }
-
- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);
- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);
- }
-
- *s = hsum_float_8(acc);
-
-#elif defined __riscv_v_intrinsic
-
- float sumf = 0;
- for (int i = 0; i < nb; ++i) {
-
- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
-
- const uint8_t * restrict q6 = x[i].ql;
- const uint8_t * restrict qh = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
-
- const int8_t * restrict scale = x[i].scales;
-
- size_t vl;
-
- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
-
- int sum_t = 0;
- int is = 0;
-
- for (int j = 0; j < QK_K/128; ++j) {
-
- vl = 32;
-
- // load qh
- vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);
-
- // load Q6
- vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);
- vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);
-
- vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);
- vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);
- vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);
- vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);
-
- vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);
- vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);
- vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);
- vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);
-
- vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);
- vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);
- vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);
- vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);
-
- vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);
- vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);
- vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);
- vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);
-
- // load Q8 and take product
- vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);
- vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);
- vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);
- vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);
-
- vl = 16;
-
- vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);
- vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);
- vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);
- vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);
- vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);
- vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);
- vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);
- vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);
-
- vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);
- vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);
- vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);
- vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);
-
- sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);
-
- q6 += 64; qh += 32; q8 += 128; is=8;
-
- }
-
- sumf += d * sum_t;
-
- }
-
- *s = sumf;
-
-#else
-
- int8_t aux8[QK_K];
- int16_t aux16[8];
- float sums [8];
- int32_t aux32[8];
- memset(sums, 0, 8*sizeof(float));
-
- float sumf = 0;
- for (int i = 0; i < nb; ++i) {
- const uint8_t * restrict q4 = x[i].ql;
- const uint8_t * restrict qh = x[i].qh;
- const int8_t * restrict q8 = y[i].qs;
- memset(aux32, 0, 8*sizeof(int32_t));
- int8_t * restrict a = aux8;
- for (int j = 0; j < QK_K; j += 128) {
- for (int l = 0; l < 32; ++l) {
- a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
- a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
- a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
- a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
- }
- a += 128;
- q4 += 64;
- qh += 32;
- }
- a = aux8;
- int is = 0;
- for (int j = 0; j < QK_K/16; ++j) {
- int scale = x[i].scales[is++];
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
- q8 += 8; a += 8;
- }
- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
- }
- for (int l = 0; l < 8; ++l) sumf += sums[l];
- *s = sumf;
-#endif
-}
diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc
deleted file mode 100644
index 46961faf..00000000
--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc
+++ /dev/null
@@ -1,169 +0,0 @@
-static void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
- const int qk = QK8_0;
- const int nb = n / qk;
-
- assert(n % qk == 0);
-#if defined(__ARM_FEATURE_MATMUL_INT8)
- assert((nrc == 2) || (nrc == 1));
-#else
- assert(nrc == 1);
-#endif
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
- const block_q8_0 * restrict x = vx;
- const block_q8_0 * restrict y = vy;
-
-#if defined(__ARM_FEATURE_MATMUL_INT8)
- if (nrc == 2) {
- const block_q8_0 * restrict vx0 = vx;
- const block_q8_0 * restrict vx1 = vx + bx;
- const block_q8_0 * restrict vy0 = vy;
- const block_q8_0 * restrict vy1 = vy + by;
-
- float32x4_t sumv0 = vdupq_n_f32(0.0f);
-
- for (int i = 0; i < nb; i++) {
- const block_q8_0 * restrict b_x0 = &vx0[i];
- const block_q8_0 * restrict b_y0 = &vy0[i];
-
- const block_q8_0 * restrict b_x1 = &vx1[i];
- const block_q8_0 * restrict b_y1 = &vy1[i];
-
- const int8x16_t x0_l = vld1q_s8(b_x0->qs);
- const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16);
- const int8x16_t x1_l = vld1q_s8(b_x1->qs);
- const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16);
-
- // load y
- const int8x16_t y0_l = vld1q_s8(b_y0->qs);
- const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
- const int8x16_t y1_l = vld1q_s8(b_y1->qs);
- const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
-
- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
- GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
-
- int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
- int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
-
- int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
- int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
-
- int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
- int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
-
- int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
- int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
-
- sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
- l1, r1)), l2, r2)), l3, r3))), scale);
- }
- float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
- float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
-
- vst1_f32(s, vget_low_f32(sumv2));
- vst1_f32(s + bs, vget_high_f32(sumv2));
- return;
- }
-#endif
-#if defined(__ARM_NEON)
- float32x4_t sumv0 = vdupq_n_f32(0.0f);
- float32x4_t sumv1 = vdupq_n_f32(0.0f);
-
- assert(nb % 2 == 0); // TODO: handle odd nb
-
- for (int i = 0; i < nb; i += 2) {
- const block_q8_0 * restrict x0 = &x[i + 0];
- const block_q8_0 * restrict x1 = &x[i + 1];
- const block_q8_0 * restrict y0 = &y[i + 0];
- const block_q8_0 * restrict y1 = &y[i + 1];
-
- const int8x16_t x0_0 = vld1q_s8(x0->qs);
- const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);
- const int8x16_t x1_0 = vld1q_s8(x1->qs);
- const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);
-
- // load y
- const int8x16_t y0_0 = vld1q_s8(y0->qs);
- const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);
- const int8x16_t y1_0 = vld1q_s8(y1->qs);
- const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);
-
- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(
- ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),
- ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
-
- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(
- ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),
- ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
- }
-
- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
-#elif defined(__AVX2__) || defined(__AVX__)
- // Initialize accumulator with zeros
- __m256 acc = _mm256_setzero_ps();
-
- // Main loop
- for (int i = 0; i < nb; ++i) {
- // Compute combined scale for the block
- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
- __m256i qx = _mm256_loadu_si256((const __m256i *)x[i].qs);
- __m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
-
-#ifdef __AVX2__
- const __m256 q = mul_sum_i8_pairs_float_avx2(qx, qy);
-#else
- const __m256 q = mul_sum_i8_pairs_float_avx(qx, qy);
-#endif
-
- // Multiply q with scale and accumulate
-#if defined(__AVX2__)
- acc = _mm256_fmadd_ps( d, q, acc );
-#else
- acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );
-#endif
- }
-
- *s = hsum_float_8(acc);
-#elif defined(__riscv_v_intrinsic)
- float sumf = 0.0;
- size_t vl = __riscv_vsetvl_e8m1(qk);
-
- for (int i = 0; i < nb; i++) {
- // load elements
- vint8m1_t bx_0 = __riscv_vle8_v_i8m1(x[i].qs, vl);
- vint8m1_t by_0 = __riscv_vle8_v_i8m1(y[i].qs, vl);
-
- vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx_0, by_0, vl);
-
- vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);
- vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl);
-
- int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);
-
- sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));
- }
-
- *s = sumf;
-#else
- // scalar
- float sumf = 0.0;
-
- for (int i = 0; i < nb; i++) {
- int sumi = 0;
-
- for (int j = 0; j < qk; j++) {
- sumi += x[i].qs[j]*y[i].qs[j];
- }
-
- sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));
- }
-
- *s = sumf;
-#endif
-}
diff --git a/llama.cpp/imatrix/imatrix.cpp b/llama.cpp/imatrix/imatrix.cpp
index 74576f01..18dd537e 100644
--- a/llama.cpp/imatrix/imatrix.cpp
+++ b/llama.cpp/imatrix/imatrix.cpp
@@ -27,6 +27,7 @@ struct Stats {
};
struct StatParams {
+ std::string dataset;
std::string ofile = "imatrix.dat";
int n_output_frequency = 10;
int verbosity = 1;
@@ -50,7 +51,7 @@ private:
std::vector<float> m_src1_data;
std::vector<char> m_ids; // the expert ids from ggml_mul_mat_id
//
- void save_imatrix(const char * file_name) const;
+ void save_imatrix(const char * file_name, const char * dataset) const;
void keep_imatrix(int ncall) const;
};
@@ -203,7 +204,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void *
}
void IMatrixCollector::save_imatrix() const {
- save_imatrix(m_params.ofile.empty() ? "imatrix.dat" : m_params.ofile.c_str());
+ save_imatrix(m_params.ofile.empty() ? "imatrix.dat" : m_params.ofile.c_str(), m_params.dataset.c_str());
}
void IMatrixCollector::keep_imatrix(int ncall) const {
@@ -211,24 +212,33 @@ void IMatrixCollector::keep_imatrix(int ncall) const {
if (file_name.empty()) file_name = "imatrix.dat";
file_name += ".at_";
file_name += std::to_string(ncall);
- save_imatrix(file_name.c_str());
+ save_imatrix(file_name.c_str(), m_params.dataset.c_str());
}
-void IMatrixCollector::save_imatrix(const char * fname) const {
+void IMatrixCollector::save_imatrix(const char * fname, const char * dataset) const {
std::ofstream out(fname, std::ios::binary);
int n_entries = m_stats.size();
- out.write((const char*)&n_entries, sizeof(n_entries));
- for (auto& p : m_stats) {
+ out.write((const char *) &n_entries, sizeof(n_entries));
+ for (const auto & p : m_stats) {
int len = p.first.size();
- out.write((const char*)&len, sizeof(len));
+ out.write((const char *) &len, sizeof(len));
out.write(p.first.c_str(), len);
- out.write((const char*)&p.second.ncall, sizeof(p.second.ncall));
+ out.write((const char *) &p.second.ncall, sizeof(p.second.ncall));
int nval = p.second.values.size();
- out.write((const char*)&nval, sizeof(nval));
- if (nval > 0) out.write((const char*)p.second.values.data(), nval*sizeof(float));
+ out.write((const char *) &nval, sizeof(nval));
+ if (nval > 0) out.write((const char *) p.second.values.data(), nval * sizeof(float));
}
+
+ // Write the number of call the matrix was computed with
+ out.write((const char *) &m_last_call, sizeof(m_last_call));
+
+ // Write the dataset name at the end of the file to later on specify it in quantize
+ int n_dataset = strlen(dataset);
+ out.write((const char *) &n_dataset, sizeof(n_dataset));
+ out.write(dataset, n_dataset);
+
if (m_params.verbosity > 0) {
- fprintf(stderr, "\n%s: stored collected data after %d chunks in %s\n",__func__,m_last_call,fname);
+ fprintf(stderr, "\n%s: stored collected data after %d chunks in %s\n", __func__, m_last_call, fname);
}
}
@@ -565,6 +575,29 @@ int main(int argc, char ** argv) {
}
}
+ gpt_params params;
+ params.n_batch = 512;
+ if (!gpt_params_parse(args.size(), args.data(), params)) {
+ return 1;
+ }
+
+ params.logits_all = true;
+ params.n_batch = std::min(params.n_batch, params.n_ctx);
+
+ print_build_info();
+
+ if (params.seed == LLAMA_DEFAULT_SEED) {
+ params.seed = time(NULL);
+ }
+
+ fprintf(stderr, "%s: seed = %u\n", __func__, params.seed);
+
+ std::mt19937 rng(params.seed);
+ if (params.random_prompt) {
+ params.prompt = gpt_random_prompt(rng);
+ }
+
+ sparams.dataset = params.prompt_file;
g_collector.set_parameters(std::move(sparams));
if (!combine_files.empty()) {
@@ -603,28 +636,6 @@ int main(int argc, char ** argv) {
}
}
- gpt_params params;
- params.n_batch = 512;
- if (!gpt_params_parse(args.size(), args.data(), params)) {
- return 1;
- }
-
- params.logits_all = true;
- params.n_batch = std::min(params.n_batch, params.n_ctx);
-
- print_build_info();
-
- if (params.seed == LLAMA_DEFAULT_SEED) {
- params.seed = time(NULL);
- }
-
- fprintf(stderr, "%s: seed = %u\n", __func__, params.seed);
-
- std::mt19937 rng(params.seed);
- if (params.random_prompt) {
- params.prompt = gpt_random_prompt(rng);
- }
-
llama_backend_init();
llama_numa_init(params.numa);
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index c788c059..58ed01a4 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -21,6 +21,7 @@
#endif
#include <algorithm>
+#include <future>
#include <array>
#include <cassert>
#include <cinttypes>
@@ -268,6 +269,7 @@ enum llm_kv {
LLM_KV_SSM_TIME_STEP_RANK,
LLM_KV_TOKENIZER_MODEL,
+ LLM_KV_TOKENIZER_PRE,
LLM_KV_TOKENIZER_LIST,
LLM_KV_TOKENIZER_TOKEN_TYPE,
LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,
@@ -344,6 +346,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },
{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },
+ { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },
{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },
{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },
{ LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },
@@ -1645,7 +1648,7 @@ struct llama_hparams {
float f_logit_scale = 0.0f;
bool causal_attn = true;
- bool need_kq_pos = false;
+ bool use_alibi = false; // currently, we need KQ_pos data for ALiBi-based models
enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE;
enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE;
@@ -1735,6 +1738,7 @@ struct llama_cparams {
bool embeddings;
bool causal_attn;
bool offload_kqv;
+ bool flash_attn;
enum llama_pooling_type pooling_type;
@@ -1838,8 +1842,8 @@ struct llama_kv_cache {
bool has_shift = false;
bool do_defrag = false;
bool do_copy = false;
- // with recurrent state models, a cell can hold the state for more than one past token
- bool recurrent = false;
+ bool recurrent = false; // with recurrent state models, a cell can hold the state for more than one past token
+ bool v_trans = true; // the value tensor is transposed
// Note: The value of head isn't only used to optimize searching
// for a free KV slot. llama_decode_internal also uses it, so it
@@ -1916,7 +1920,8 @@ struct llama_vocab {
ttype type;
};
- enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;
+ enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;
+ enum llama_vocab_pre_type type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
std::unordered_map<token, id> token_to_id;
std::vector<token_data> id_to_token;
@@ -2137,11 +2142,14 @@ struct llama_context {
static bool llama_kv_cache_init(
struct llama_kv_cache & cache,
- const llama_model & model,
+ const llama_context * ctx,
ggml_type type_k,
ggml_type type_v,
uint32_t kv_size,
bool offload) {
+ const llama_model & model = ctx->model;
+ const llama_cparams & cparams = ctx->cparams;
+
const struct llama_hparams & hparams = model.hparams;
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s();
@@ -2152,6 +2160,7 @@ static bool llama_kv_cache_init(
// TODO: find a nicer way to add other recurrent model architectures
cache.recurrent = model.arch == LLM_ARCH_MAMBA;
+ cache.v_trans = !cparams.flash_attn;
// TODO: support mixed reccurent Transformer architectues
// NOTE: (!a || b) is a logical implication (a -> b)
@@ -2364,6 +2373,10 @@ static void llama_kv_cache_clear(struct llama_kv_cache & cache) {
}
cache.head = 0;
cache.used = 0;
+
+ for (auto & buf : cache.bufs) {
+ ggml_backend_buffer_clear(buf, 0);
+ }
}
static bool llama_kv_cache_seq_rm(
@@ -2684,6 +2697,7 @@ namespace GGUFMeta {
case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";
case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";
case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";
+ case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";
}
return "unknown";
}
@@ -2695,13 +2709,16 @@ namespace GGUFMeta {
__func__, override_type_to_str(ovrd->tag), ovrd->key);
switch (ovrd->tag) {
case LLAMA_KV_OVERRIDE_TYPE_BOOL: {
- LLAMA_LOG_INFO("%s\n", ovrd->bool_value ? "true" : "false");
+ LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");
} break;
case LLAMA_KV_OVERRIDE_TYPE_INT: {
- LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->int_value);
+ LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);
} break;
case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {
- LLAMA_LOG_INFO("%.6f\n", ovrd->float_value);
+ LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);
+ } break;
+ case LLAMA_KV_OVERRIDE_TYPE_STR: {
+ LLAMA_LOG_INFO("%s\n", ovrd->val_str);
} break;
default:
// Shouldn't be possible to end up here, but just in case...
@@ -2720,7 +2737,7 @@ namespace GGUFMeta {
static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type
try_override(OT & target, const struct llama_model_kv_override * ovrd) {
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {
- target = ovrd->bool_value;
+ target = ovrd->val_bool;
return true;
}
return false;
@@ -2730,7 +2747,7 @@ namespace GGUFMeta {
static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type
try_override(OT & target, const struct llama_model_kv_override * ovrd) {
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {
- target = ovrd->int_value;
+ target = ovrd->val_i64;
return true;
}
return false;
@@ -2740,7 +2757,7 @@ namespace GGUFMeta {
static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type
try_override(T & target, const struct llama_model_kv_override * ovrd) {
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {
- target = ovrd->float_value;
+ target = ovrd->val_f64;
return true;
}
return false;
@@ -2749,12 +2766,11 @@ namespace GGUFMeta {
template<typename OT>
static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type
try_override(T & target, const struct llama_model_kv_override * ovrd) {
- (void)target;
- (void)ovrd;
- if (!ovrd) { return false; }
- // Currently, we should never end up here so it would be a bug if we do.
- throw std::runtime_error(format("Unsupported attempt to override string type for metadata key %s\n",
- ovrd ? ovrd->key : "NULL"));
+ if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {
+ target = ovrd->val_str;
+ return true;
+ }
+ return false;
}
static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
@@ -2787,6 +2803,7 @@ struct llama_model_loader {
size_t n_bytes = 0;
bool use_mmap = false;
+ bool check_tensors;
llama_files files;
llama_ftype ftype;
@@ -2820,7 +2837,7 @@ struct llama_model_loader {
std::string arch_name;
LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);
- llama_model_loader(const std::string & fname, bool use_mmap, const struct llama_model_kv_override * param_overrides_p) {
+ llama_model_loader(const std::string & fname, bool use_mmap, bool check_tensors, const struct llama_model_kv_override * param_overrides_p) {
int trace = 0;
if (getenv("LLAMA_TRACE")) {
trace = atoi(getenv("LLAMA_TRACE"));
@@ -2919,9 +2936,17 @@ struct llama_model_loader {
fver = (enum llama_fver) gguf_get_version(meta);
+ std::set<std::string> tensor_names;
for (auto & w : weights) {
n_elements += ggml_nelements(w.tensor);
n_bytes += ggml_nbytes(w.tensor);
+ // make sure there is no duplicated tensor names
+ const std::string name(w.tensor->name);
+ auto found = tensor_names.find(name);
+ if (found != tensor_names.end()) {
+ throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", w.tensor->name));
+ }
+ tensor_names.insert(name);
}
LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",
@@ -3028,6 +3053,7 @@ struct llama_model_loader {
}
this->use_mmap = use_mmap;
+ this->check_tensors = check_tensors;
}
~llama_model_loader() {
@@ -3286,6 +3312,10 @@ struct llama_model_loader {
file->seek(w.offs, SEEK_SET);
file->read_raw(cur->data, ggml_nbytes(cur));
}
+
+ if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {
+ throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
+ }
}
size_t size_done = 0;
@@ -3302,6 +3332,8 @@ struct llama_model_loader {
GGML_ASSERT(size_data != 0 && "call init_mappings() first");
std::vector<no_init<uint8_t>> read_buf;
+ std::vector<std::future<std::pair<ggml_tensor *, bool>>> validation_result;
+
for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {
const auto * weight = get_weight(ggml_get_name(cur));
if (weight == nullptr) {
@@ -3323,37 +3355,66 @@ struct llama_model_loader {
if (bufs_mmap.count(weight->idx)) {
buf_mmap = bufs_mmap.at(weight->idx);
}
+ uint8_t * data = (uint8_t *) mapping->addr + weight->offs;
+
+ if (check_tensors) {
+ validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {
+ return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size));
+ }));
+ }
+
GGML_ASSERT(buf_mmap || cur->data); // either we have a buffer to allocate the tensor in, or it is already allocated
if (buf_mmap && cur->data == nullptr) {
- ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + weight->offs);
+ ggml_backend_tensor_alloc(buf_mmap, cur, data);
if (lmlocks) {
const auto & lmlock = lmlocks->at(weight->idx);
- lmlock->grow_to(weight->offs + ggml_nbytes(cur));
+ lmlock->grow_to(weight->offs + n_size);
}
auto & mmap_used = mmaps_used[weight->idx];
mmap_used.first = std::min(mmap_used.first, weight->offs);
mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);
} else {
- ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + weight->offs, 0, n_size);
+ ggml_backend_tensor_set(cur, data, 0, n_size);
}
} else {
GGML_ASSERT(weight->idx < files.size());
const auto & file = files.at(weight->idx);
if (ggml_backend_buffer_is_host(cur->buffer)) {
file->seek(weight->offs, SEEK_SET);
- file->read_raw(cur->data, ggml_nbytes(cur));
+ file->read_raw(cur->data, n_size);
+ if (check_tensors) {
+ validation_result.emplace_back(std::async(std::launch::async, [cur, n_size] {
+ return std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size));
+ }));
+ }
} else {
- read_buf.resize(ggml_nbytes(cur));
+ read_buf.resize(n_size);
file->seek(weight->offs, SEEK_SET);
- file->read_raw(read_buf.data(), ggml_nbytes(cur));
+ file->read_raw(read_buf.data(), n_size);
ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
+ if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {
+ throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
+ }
}
}
size_done += n_size;
}
+ // check validation results
+ bool validation_failed = false;
+ for (auto & future : validation_result) {
+ auto result = future.get();
+ if (!result.second) {
+ LLAMA_LOG_ERROR("%s: tensor '%s' has invalid data\n", __func__, ggml_get_name(result.first));
+ validation_failed = true;
+ }
+ }
+ if (validation_failed) {
+ throw std::runtime_error("found tensors with invalid data");
+ }
+
// check if this is the last call and do final cleanup
if (size_done >= size_data) {
// unmap offloaded tensors and metadata
@@ -3948,7 +4009,7 @@ static void llm_load_hparams(
model.ftype = ml.ftype;
if (hparams.f_max_alibi_bias > 0.0f) {
- hparams.need_kq_pos = true;
+ hparams.use_alibi = true;
}
hparams.rope_type = llama_rope_type(&model);
@@ -3971,11 +4032,13 @@ static void llm_load_vocab(
// determine vocab type
{
- std::string tokenizer_name;
+ std::string tokenizer_model;
+ std::string tokenizer_pre;
- ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_name);
+ ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_model);
+ ml.get_key(LLM_KV_TOKENIZER_PRE, tokenizer_pre, false);
- if (tokenizer_name == "no_vocab") {
+ if (tokenizer_model == "no_vocab") {
vocab.type = LLAMA_VOCAB_TYPE_NONE;
// default special tokens
@@ -3989,7 +4052,7 @@ static void llm_load_vocab(
vocab.linefeed_id = -1;
return;
- } else if (tokenizer_name == "llama") {
+ } else if (tokenizer_model == "llama") {
vocab.type = LLAMA_VOCAB_TYPE_SPM;
// default special tokens
@@ -4034,9 +4097,27 @@ static void llm_load_vocab(
if (add_space_prefix_keyidx != -1) {
vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
} // The default value of add_space_prefix is true.
- } else if (tokenizer_name == "gpt2") {
- vocab.type = LLAMA_VOCAB_TYPE_BPE;
+ } else if (tokenizer_model == "bert") {
+ vocab.type = LLAMA_VOCAB_TYPE_WPM;
+ // default special tokens
+ vocab.special_bos_id = -1;
+ vocab.special_eos_id = -1;
+ vocab.special_unk_id = 100;
+ vocab.special_sep_id = 102;
+ vocab.special_pad_id = 0;
+ vocab.special_cls_id = 101;
+ vocab.special_mask_id = 103;
+ vocab.add_space_prefix = false;
+ } else {
+ if (tokenizer_model == "gpt2") {
+ vocab.type = LLAMA_VOCAB_TYPE_BPE;
+ } else {
+ LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_model.c_str());
+ LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__);
+ vocab.type = LLAMA_VOCAB_TYPE_SPM;
+ return;
+ }
// read bpe merges and populate bpe ranks
const int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());
if (merges_keyidx == -1) {
@@ -4070,23 +4151,50 @@ static void llm_load_vocab(
vocab.special_pad_id = -1;
vocab.special_cls_id = -1;
vocab.special_mask_id = -1;
- } else if (tokenizer_name == "bert") {
- vocab.type = LLAMA_VOCAB_TYPE_WPM;
+ }
- // default special tokens
- vocab.special_bos_id = -1;
- vocab.special_eos_id = -1;
- vocab.special_unk_id = 100;
- vocab.special_sep_id = 102;
- vocab.special_pad_id = 0;
- vocab.special_cls_id = 101;
- vocab.special_mask_id = 103;
- vocab.add_space_prefix = false;
+ // for now, only BPE models have pre-tokenizers
+ if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
+ if (tokenizer_pre.empty()) {
+ LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
+ LLAMA_LOG_WARN("%s: \n", __func__);
+ LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
+ LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED! \n", __func__);
+ LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL \n", __func__);
+ LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
+ LLAMA_LOG_WARN("%s: \n", __func__);
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ } else if (
+ tokenizer_pre == "default") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+ } else if (
+ tokenizer_pre == "llama3" ||
+ tokenizer_pre == "llama-v3" ||
+ tokenizer_pre == "llama-bpe") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
+ } else if (
+ tokenizer_pre == "deepseek-llm") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;
+ } else if (
+ tokenizer_pre == "deepseek-coder") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER;
+ } else if (
+ tokenizer_pre == "falcon") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_FALCON;
+ } else if (
+ tokenizer_pre == "mpt") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_MPT;
+ } else if (
+ tokenizer_pre == "starcoder") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER;
+ } else if (
+ tokenizer_pre == "gpt-2") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
+ } else {
+ throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
+ }
} else {
- LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_name.c_str());
- LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__);
-
- vocab.type = LLAMA_VOCAB_TYPE_SPM;
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
}
}
@@ -5781,7 +5889,7 @@ static bool llm_load_tensors(
// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback
static int llama_model_load(const std::string & fname, llama_model & model, llama_model_params & params) {
try {
- llama_model_loader ml(fname, params.use_mmap, params.kv_overrides);
+ llama_model_loader ml(fname, params.use_mmap, params.check_tensors, params.kv_overrides);
model.hparams.vocab_only = params.vocab_only;
@@ -5911,37 +6019,47 @@ static struct ggml_tensor * llm_build_inp_embd(
static void llm_build_kv_store(
struct ggml_context * ctx,
const llama_hparams & hparams,
+ const llama_cparams & cparams,
const llama_kv_cache & kv,
struct ggml_cgraph * graph,
struct ggml_tensor * k_cur,
struct ggml_tensor * v_cur,
- int64_t n_ctx,
int32_t n_tokens,
int32_t kv_head,
const llm_build_cb & cb,
int64_t il) {
+ const int64_t n_ctx = cparams.n_ctx;
+
const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();
const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(kv.size == n_ctx);
- // compute the transposed [n_tokens, n_embd] V matrix
- assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens);
- struct ggml_tensor * v_cur_t = ggml_transpose(ctx, v_cur);
- cb(v_cur_t, "v_cur_t", il);
-
struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k_l[il], n_tokens*n_embd_k_gqa,
(ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa))*kv_head);
cb(k_cache_view, "k_cache_view", il);
- struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_v_gqa,
- ( n_ctx)*ggml_element_size(kv.v_l[il]),
- (kv_head)*ggml_element_size(kv.v_l[il]));
+ // note: storing RoPE-ed version of K in the KV cache
+ ggml_build_forward_expand(graph, ggml_cpy(ctx, k_cur, k_cache_view));
+
+ assert(v_cur->ne[0] == n_embd_v_gqa && v_cur->ne[1] == n_tokens);
+
+ struct ggml_tensor * v_cache_view = nullptr;
+
+ if (cparams.flash_attn) {
+ v_cache_view = ggml_view_1d(ctx, kv.v_l[il], n_tokens*n_embd_v_gqa,
+ (kv_head)*ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa));
+ } else {
+ // note: the V cache is transposed when not using flash attention
+ v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_v_gqa,
+ ( n_ctx)*ggml_element_size(kv.v_l[il]),
+ (kv_head)*ggml_element_size(kv.v_l[il]));
+
+ v_cur = ggml_transpose(ctx, v_cur);
+ }
cb(v_cache_view, "v_cache_view", il);
- // important: storing RoPE-ed version of K in the KV cache!
- ggml_build_forward_expand(graph, ggml_cpy(ctx, k_cur, k_cache_view));
- ggml_build_forward_expand(graph, ggml_cpy(ctx, v_cur_t, v_cache_view));
+ ggml_build_forward_expand(graph, ggml_cpy(ctx, v_cur, v_cache_view));
}
static struct ggml_tensor * llm_build_norm(
@@ -6161,11 +6279,11 @@ static struct ggml_tensor * llm_build_moe_ffn(
return moe_out;
}
-// if max_alibi_bias > 0 then apply ALiBi
static struct ggml_tensor * llm_build_kqv(
struct ggml_context * ctx,
const llama_model & model,
const llama_hparams & hparams,
+ const llama_cparams & cparams,
const llama_kv_cache & kv,
struct ggml_cgraph * graph,
struct ggml_tensor * wo,
@@ -6173,12 +6291,12 @@ static struct ggml_tensor * llm_build_kqv(
struct ggml_tensor * q_cur,
struct ggml_tensor * kq_mask,
struct ggml_tensor * kq_pos,
- int64_t n_ctx,
int32_t n_tokens,
int32_t n_kv,
float kq_scale,
const llm_build_cb & cb,
int il) {
+ const int64_t n_ctx = cparams.n_ctx;
const int64_t n_head = hparams.n_head;
const int64_t n_head_kv = hparams.n_head_kv;
const int64_t n_embd_head_k = hparams.n_embd_head_k;
@@ -6196,71 +6314,99 @@ static struct ggml_tensor * llm_build_kqv(
0);
cb(k, "k", il);
- struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);
- cb(kq, "kq", il);
+ struct ggml_tensor * cur;
- if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) {
- // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs
- // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847
- ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
- }
+ if (cparams.flash_attn) {
+ GGML_UNUSED(model);
+ GGML_UNUSED(n_ctx);
- if (model.arch == LLM_ARCH_GROK) {
- // need to do the following:
- // multiply by attn_output_multiplyer of 0.08838834764831845
- // and then :
- // kq = 30 * tanh(kq / 30)
- // before the softmax below
+ // note: if this assert triggers, then some check has failed earlier
+ // the idea is to detect during context creation that ALiBi would be used and disable Flash Attention
+ GGML_ASSERT(kq_pos == nullptr && "ALiBi is not yet supported with Flash Attention");
- //try from phi2
- //ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+ // split cached v into n_head heads (not transposed)
+ struct ggml_tensor * v =
+ ggml_view_3d(ctx, kv.v_l[il],
+ n_embd_head_v, n_kv, n_head_kv,
+ ggml_row_size(kv.v_l[il]->type, n_embd_k_gqa),
+ ggml_row_size(kv.v_l[il]->type, n_embd_head_k),
+ 0);
+ cb(v, "v", il);
- kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f));
- kq = ggml_scale(ctx, kq, 30);
- }
+ cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale);
+
+ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) {
+ ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);
+ }
+
+ cur = ggml_reshape_2d(ctx, cur, n_embd_head_k*n_head, n_tokens);
+ } else {
+ struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);
+ cb(kq, "kq", il);
+
+ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) {
+ // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs
+ // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847
+ ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+ }
+
+ if (model.arch == LLM_ARCH_GROK) {
+ // need to do the following:
+ // multiply by attn_output_multiplyer of 0.08838834764831845
+ // and then :
+ // kq = 30 * tanh(kq / 30)
+ // before the softmax below
+
+ //try from phi2
+ //ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+
+ kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f));
+ kq = ggml_scale(ctx, kq, 30);
+ }
#if defined(GGML_USE_KOMPUTE)
#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Kompute")
#pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024")
#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488")
- if (hparams.f_max_alibi_bias > 0.0f) {
- kq = ggml_scale(ctx, kq, kq_scale);
- cb(kq, "kq_scaled", il);
+ if (hparams.use_alibi) {
+ kq = ggml_scale(ctx, kq, kq_scale);
+ cb(kq, "kq_scaled", il);
- kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, hparams.f_max_alibi_bias);
- cb(kq, "kq_scaled_alibi", il);
+ kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, hparams.f_max_alibi_bias);
+ cb(kq, "kq_scaled_alibi", il);
- kq = ggml_add(ctx, kq, kq_mask);
- cb(kq, "kq_masked", il);
+ kq = ggml_add(ctx, kq, kq_mask);
+ cb(kq, "kq_masked", il);
- kq = ggml_soft_max(ctx, kq);
- cb(kq, "kq_soft_max", il);
- } else
+ kq = ggml_soft_max(ctx, kq);
+ cb(kq, "kq_soft_max", il);
+ } else
#endif
- {
- kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_pos, kq_scale, hparams.f_max_alibi_bias);
- cb(kq, "kq_soft_max_ext", il);
- }
+ {
+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_pos, kq_scale, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
+ }
- GGML_ASSERT(kv.size == n_ctx);
+ GGML_ASSERT(kv.size == n_ctx);
- // split cached v into n_head heads
- struct ggml_tensor * v =
- ggml_view_3d(ctx, kv.v_l[il],
- n_kv, n_embd_head_v, n_head_kv,
- ggml_element_size(kv.v_l[il])*n_ctx,
- ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head_v,
- 0);
- cb(v, "v", il);
+ // split cached v into n_head heads
+ struct ggml_tensor * v =
+ ggml_view_3d(ctx, kv.v_l[il],
+ n_kv, n_embd_head_v, n_head_kv,
+ ggml_element_size(kv.v_l[il])*n_ctx,
+ ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head_v,
+ 0);
+ cb(v, "v", il);
- struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq);
- cb(kqv, "kqv", il);
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq);
+ cb(kqv, "kqv", il);
- struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3);
- cb(kqv_merged, "kqv_merged", il);
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
- struct ggml_tensor * cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens);
- cb(cur, "kqv_merged_cont", il);
+ cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
+ }
ggml_build_forward_expand(graph, cur);
@@ -6280,6 +6426,7 @@ static struct ggml_tensor * llm_build_kv(
struct ggml_context * ctx,
const llama_model & model,
const llama_hparams & hparams,
+ const llama_cparams & cparams,
const llama_kv_cache & kv,
struct ggml_cgraph * graph,
struct ggml_tensor * wo,
@@ -6289,7 +6436,6 @@ static struct ggml_tensor * llm_build_kv(
struct ggml_tensor * q_cur,
struct ggml_tensor * kq_mask,
struct ggml_tensor * kq_pos,
- int64_t n_ctx,
int32_t n_tokens,
int32_t kv_head,
int32_t n_kv,
@@ -6303,12 +6449,12 @@ static struct ggml_tensor * llm_build_kv(
ggml_build_forward_expand(graph, k_cur);
ggml_build_forward_expand(graph, v_cur);
- llm_build_kv_store(ctx, hparams, kv, graph, k_cur, v_cur, n_ctx, n_tokens, kv_head, cb, il);
+ llm_build_kv_store(ctx, hparams, cparams, kv, graph, k_cur, v_cur, n_tokens, kv_head, cb, il);
struct ggml_tensor * cur;
- cur = llm_build_kqv(ctx, model, hparams, kv, graph, wo, wo_b,
- q_cur, kq_mask, kq_pos, n_ctx, n_tokens, n_kv, kq_scale, cb, il);
+ cur = llm_build_kqv(ctx, model, hparams, cparams, kv, graph, wo, wo_b,
+ q_cur, kq_mask, kq_pos, n_tokens, n_kv, kq_scale, cb, il);
cb(cur, "kqv_out", il);
return cur;
@@ -6350,6 +6496,8 @@ struct llm_build_context {
const int32_t kv_head; // index of where we store new KV data in the cache
const int32_t n_orig_ctx;
+ const bool flash_attn;
+
const enum llama_pooling_type pooling_type;
const enum llama_rope_type rope_type;
@@ -6396,6 +6544,7 @@ struct llm_build_context {
n_outputs (worst_case ? n_tokens : lctx.n_outputs),
kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head),
n_orig_ctx (cparams.n_yarn_orig_ctx),
+ flash_attn (cparams.flash_attn),
pooling_type (cparams.pooling_type),
rope_type (hparams.rope_type),
cb (cb),
@@ -6510,15 +6659,31 @@ struct llm_build_context {
ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*id));
- ggml_tensor * view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],
- nm, n_embd_v_gqa,
- ggml_row_size(kv_self.v_l[il]->type, kv_self.size),
- ggml_row_size(kv_self.v_l[il]->type, i));
+ ggml_tensor * view_v_src;
+ ggml_tensor * view_v_dst;
+
+ if (flash_attn) {
+ // NOTE: the V cache is not transposed when using flash attention
+ view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],
+ n_embd_v_gqa, nm,
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa),
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*i));
- ggml_tensor * view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],
- nm, n_embd_v_gqa,
- ggml_row_size(kv_self.v_l[il]->type, kv_self.size),
- ggml_row_size(kv_self.v_l[il]->type, id));
+ view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],
+ n_embd_v_gqa, nm,
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa),
+ ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*id));
+ } else {
+ view_v_src = ggml_view_2d(ctx0, kv_self.v_l[il],
+ nm, n_embd_v_gqa,
+ ggml_row_size(kv_self.v_l[il]->type, kv_self.size),
+ ggml_row_size(kv_self.v_l[il]->type, i));
+
+ view_v_dst = ggml_view_2d(ctx0, kv_self.v_l[il],
+ nm, n_embd_v_gqa,
+ ggml_row_size(kv_self.v_l[il]->type, kv_self.size),
+ ggml_row_size(kv_self.v_l[il]->type, id));
+ }
ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_k_src, view_k_dst));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, view_v_src, view_v_dst));
@@ -6548,20 +6713,26 @@ struct llm_build_context {
struct ggml_tensor * build_inp_KQ_mask(bool causal = true) {
if (causal) {
- lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_tokens);
+ lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD));
} else {
- lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens);
+ lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD));
}
cb(lctx.inp_KQ_mask, "KQ_mask", -1);
ggml_set_input(lctx.inp_KQ_mask);
- return lctx.inp_KQ_mask;
+ return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_mask, GGML_TYPE_F16) : lctx.inp_KQ_mask;
}
- struct ggml_tensor * build_inp_KQ_pos() {
- lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_kv);
+ struct ggml_tensor * build_inp_KQ_pos(bool causal = true) {
+ if (causal) {
+ lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_kv);
+ } else {
+ // TODO: this will be needed for ALiBi-based BERT models
+ // https://github.com/ggerganov/llama.cpp/pull/6826
+ lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_tokens);
+ }
cb(lctx.inp_KQ_pos, "KQ_pos", -1);
ggml_set_input(lctx.inp_KQ_pos);
- return lctx.inp_KQ_pos;
+ return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_pos, GGML_TYPE_F16) : lctx.inp_KQ_pos;
}
struct ggml_tensor * build_inp_mean() {
@@ -6667,9 +6838,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -6807,9 +6978,9 @@ struct llm_build_context {
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -6914,9 +7085,9 @@ struct llm_build_context {
ext_factor, attn_factor, beta_fast, beta_slow
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -7034,9 +7205,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -7159,9 +7330,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);
}
if (il == n_layer - 1) {
@@ -7311,9 +7482,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
- model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
+ model.layers[il].wo, NULL,
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -7423,9 +7594,9 @@ struct llm_build_context {
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -7627,9 +7798,9 @@ struct llm_build_context {
);
cb(Vcur, "Vcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Q, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Q, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -7723,9 +7894,9 @@ struct llm_build_context {
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
cb(Qcur, "Qcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -8016,9 +8187,9 @@ struct llm_build_context {
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -8147,14 +8318,15 @@ struct llm_build_context {
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
- model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
} else {
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
}
@@ -8296,9 +8468,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -8414,9 +8586,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -8527,9 +8699,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -8641,9 +8813,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -8796,9 +8968,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);
}
if (il == n_layer - 1) {
@@ -8913,9 +9085,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
- model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);
}
if (il == n_layer - 1) {
@@ -9026,9 +9198,9 @@ struct llm_build_context {
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
struct ggml_tensor * sa_out = cur;
@@ -9129,9 +9301,9 @@ struct llm_build_context {
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -9236,9 +9408,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -9352,9 +9524,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -9469,9 +9641,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -9599,9 +9771,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -9720,9 +9892,9 @@ struct llm_build_context {
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, NULL,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);
}
if (il == n_layer - 1) {
@@ -9839,9 +10011,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -10129,9 +10301,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -10260,9 +10432,9 @@ struct llm_build_context {
);
cb(Kcur, "Kcur", il);
- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
+ cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
model.layers[il].wo, nullptr,
- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
}
if (il == n_layer - 1) {
@@ -10689,7 +10861,9 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {
}
}
- if (hparams.need_kq_pos) {
+ // ALiBi requires the KQ_pos tensor to provide the sequence position of each token in the batch
+ // this allows to process multiple sequences in parallel with ALiBi-based models
+ if (hparams.use_alibi) {
const int64_t n_kv = kv_self.n;
GGML_ASSERT(lctx.inp_KQ_pos);
@@ -11071,7 +11245,7 @@ static int llama_decode_internal(
// a heuristic, to avoid attending the full cache if it is not yet utilized
// after enough generations, the benefit from this heuristic disappears
// if we start defragmenting the cache, the benefit from this will be more important
- kv_self.n = std::min(kv_self.size, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32)));
+ kv_self.n = std::min(kv_self.size, std::max(256u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 256)));
//kv_self.n = llama_kv_cache_cell_max(kv_self);
}
}
@@ -11239,6 +11413,10 @@ static int llama_decode_internal(
}
}
+ // Reset state for the next token before backend sync, to allow the CPU activities in the reset to
+ // overlap with device computation.
+ ggml_backend_sched_reset(lctx.sched);
+
return 0;
}
@@ -11264,7 +11442,9 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {
// each move requires 6*n_layer tensors (see build_defrag)
// - source view, destination view, copy operation
// - x2 for keys and values
- const uint32_t max_moves = LLAMA_MAX_NODES/(6*n_layer);
+ //const uint32_t max_moves = LLAMA_MAX_NODES/(6*n_layer);
+ // TODO: tmp fix https://github.com/ggerganov/llama.cpp/issues/6685#issuecomment-2057579516
+ const uint32_t max_moves = (LLAMA_MAX_NODES - 2*n_layer)/(6*n_layer);
// determine which KV cells to move where
//
@@ -11588,7 +11768,7 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {
}
case LLAMA_VOCAB_TYPE_BPE: {
GGML_ASSERT(false);
- return unicode_utf8_to_byte(token_data.text);
+ return unicode_utf8_to_byte(token_data.text); // TODO: why is this here after GGML_ASSERT?
}
case LLAMA_VOCAB_TYPE_WPM: {
GGML_ASSERT(false);
@@ -11810,7 +11990,79 @@ struct llm_tokenizer_bpe {
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
int final_prev_index = -1;
- auto word_collection = bpe_gpt2_preprocess(text);
+
+ std::vector<std::string> word_collection;
+ switch (vocab.type) {
+ case LLAMA_VOCAB_TYPE_BPE:
+ switch (vocab.type_pre) {
+ case LLAMA_VOCAB_PRE_TYPE_LLAMA3:
+ word_collection = unicode_regex_split(text, {
+ // original regex from tokenizer.json
+ //"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+
+ // adapted: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2080233989
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ });
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:
+ word_collection = unicode_regex_split(text, {
+ "[\r\n]",
+ "\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗA-Za-z𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+",
+ "\\s?[!-/:-~!-/:-~‘-‟ -。]+",
+ "\\s+$",
+ "[一-龥ࠀ-一가-퟿]+",
+ "\\p{N}+",
+ });
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:
+ word_collection = unicode_regex_split(text, {
+ "[\r\n]",
+ "\\s?\\p{L}+",
+ "\\s?\\p{P}+",
+ "[一-龥ࠀ-一가-퟿]+",
+ "\\p{N}+",
+ });
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_FALCON:
+ word_collection = unicode_regex_split(text, {
+ "[\\p{P}\\$\\+<=>\\^~\\|]+",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ "\\p{N}+",
+ "[0-9][0-9][0-9]",
+ });
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_MPT:
+ // TODO: MPT pre-tokenization regexes are unknown
+ // the following are close, but not exact. run the following:
+ // ./bin/test-tokenizer-0 ../models/ggml-vocab-mpt.gguf
+ GGML_ASSERT("MPT pre-tokenization regexes are unknown - fixes needed");
+ word_collection = unicode_regex_split(text, {
+ "\\s?\\p{L}+",
+ "\\s?\\p{P}+",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ });
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_STARCODER:
+ case LLAMA_VOCAB_PRE_TYPE_GPT2:
+ word_collection = unicode_regex_split(text, {
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ });
+ break;
+ default:
+ // default regex for BPE tokenization pre-processing
+ word_collection = unicode_regex_split(text, {
+ "[\\p{P}\\$\\+<=>\\^~\\|]+",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ "\\p{N}+",
+ "[0-9][0-9][0-9]",
+ });
+ break;
+ }
+ break;
+ default:
+ GGML_ASSERT(false);
+ break;
+ }
symbols_final.clear();
@@ -11937,145 +12189,6 @@ private:
work_queue.push(bigram);
}
- std::vector<std::string> bpe_gpt2_preprocess(const std::string & text) {
- std::vector<std::string> bpe_words;
- std::vector<std::string> bpe_encoded_words;
-
- std::string token = "";
- // GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
- bool collecting_numeric = false;
- bool collecting_letter = false;
- bool collecting_special = false;
- bool collecting_whitespace_lookahead = false;
- bool collecting = false;
-
- std::vector<std::string> text_utf;
- text_utf.reserve(text.size());
- bpe_words.reserve(text.size());
- bpe_encoded_words.reserve(text.size());
-
- const auto cpts = unicode_cpts_from_utf8(text);
- for (size_t i = 0; i < cpts.size(); ++i)
- text_utf.emplace_back(unicode_cpt_to_utf8(cpts[i]));
-
- for (int i = 0; i < (int)text_utf.size(); i++) {
- const std::string & utf_char = text_utf[i];
- bool split_condition = false;
- int bytes_remain = text_utf.size() - i;
- // forward backward lookups
- const std::string & utf_char_next = (i + 1 < (int)text_utf.size()) ? text_utf[i + 1] : "";
- const std::string & utf_char_next_next = (i + 2 < (int)text_utf.size()) ? text_utf[i + 2] : "";
-
- // handling contractions
- if (!split_condition && bytes_remain >= 2) {
- // 's|'t|'m|'d
- if (utf_char == "\'" && (utf_char_next == "s" || utf_char_next == "t" || utf_char_next == "m" || utf_char_next == "d")) {
- split_condition = true;
- }
- if (split_condition) {
- if (token.size()) {
- bpe_words.emplace_back(token); // push previous content as token
- }
- token = utf_char + utf_char_next;
- bpe_words.emplace_back(token);
- token = "";
- i++;
- continue;
- }
- }
- if (!split_condition && bytes_remain >= 3) {
- // 're|'ve|'ll
- if (utf_char == "\'" && (
- (utf_char_next == "r" && utf_char_next_next == "e") ||
- (utf_char_next == "v" && utf_char_next_next == "e") ||
- (utf_char_next == "l" && utf_char_next_next == "l"))
- ) {
- split_condition = true;
- }
- if (split_condition) {
- // current token + next token can be defined
- if (token.size()) {
- bpe_words.emplace_back(token); // push previous content as token
- }
- token = utf_char + utf_char_next + utf_char_next_next;
- bpe_words.emplace_back(token); // the contraction
- token = "";
- i += 2;
- continue;
- }
- }
-
- if (!split_condition && !collecting) {
- if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) {
- collecting_letter = true;
- collecting = true;
- }
- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {
- collecting_numeric = true;
- collecting = true;
- }
- else if (
- ((unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (unicode_cpt_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) ||
- (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE)
- ) {
- collecting_special = true;
- collecting = true;
- }
- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) {
- collecting_whitespace_lookahead = true;
- collecting = true;
- }
- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) {
- split_condition = true;
- }
- }
- else if (!split_condition && collecting) {
- if (collecting_letter && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER) {
- split_condition = true;
- }
- else if (collecting_numeric && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) {
- split_condition = true;
- }
- else if (collecting_special && (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) {
- split_condition = true;
- }
- else if (collecting_whitespace_lookahead && (unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {
- split_condition = true;
- }
- }
-
- if (utf_char_next == "") {
- split_condition = true; // final
- token += utf_char;
- }
-
- if (split_condition) {
- if (token.size()) {
- bpe_words.emplace_back(token);
- }
- token = utf_char;
- collecting = false;
- collecting_letter = false;
- collecting_numeric = false;
- collecting_special = false;
- collecting_whitespace_lookahead = false;
- }
- else {
- token += utf_char;
- }
- }
-
- for (std::string & word : bpe_words) {
- std::string encoded_token = "";
- for (char & c : word) {
- encoded_token += unicode_byte_to_utf8(c);
- }
- bpe_encoded_words.emplace_back(encoded_token);
- }
-
- return bpe_encoded_words;
- }
-
const llama_vocab & vocab;
std::vector<llm_symbol> symbols;
@@ -12395,7 +12508,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
} break;
case LLAMA_VOCAB_TYPE_BPE:
{
- if (add_special && vocab.special_add_bos == 1) {
+ if (add_special && vocab.special_add_bos != 0) {
GGML_ASSERT(vocab.special_bos_id != -1);
output.push_back(vocab.special_bos_id);
}
@@ -14167,14 +14280,20 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
}
static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t nrows, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {
- std::mutex mutex;
- int64_t counter = 0;
- size_t new_size = 0;
if (nthread < 2) {
// single-thread
- return ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);
+ size_t new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);
+ if (!ggml_validate_row_data(new_type, new_data, new_size)) {
+ throw std::runtime_error("quantized data validation failed");
+ }
+ return new_size;
}
- auto compute = [&mutex, &counter, &new_size, new_type, f32_data, new_data, chunk_size,
+
+ std::mutex mutex;
+ int64_t counter = 0;
+ size_t new_size = 0;
+ bool valid = true;
+ auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,
nrows, n_per_row, imatrix]() {
const int64_t nrows_per_chunk = chunk_size / n_per_row;
size_t local_size = 0;
@@ -14189,7 +14308,17 @@ static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const floa
}
lock.unlock();
const int64_t this_nrow = std::min(nrows - first_row, nrows_per_chunk);
- local_size += ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);
+ size_t this_size = ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);
+ local_size += this_size;
+
+ // validate the quantized data
+ const size_t row_size = ggml_row_size(new_type, n_per_row);
+ void * this_data = (char *) new_data + first_row * row_size;
+ if (!ggml_validate_row_data(new_type, this_data, this_size)) {
+ std::unique_lock<std::mutex> lock(mutex);
+ valid = false;
+ break;
+ }
}
};
for (int it = 0; it < nthread - 1; ++it) {
@@ -14198,6 +14327,9 @@ static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const floa
compute();
for (auto & w : workers) { w.join(); }
workers.clear();
+ if (!valid) {
+ throw std::runtime_error("quantized data validation failed");
+ }
return new_size;
}
@@ -14261,7 +14393,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
auto v = (std::vector<llama_model_kv_override>*)params->kv_overrides;
kv_overrides = v->data();
}
- llama_model_loader ml(fname_inp, use_mmap, kv_overrides);
+ llama_model_loader ml(fname_inp, use_mmap, /*check_tensors*/ true, kv_overrides);
ml.init_mappings(false); // no prefetching
llama_model model;
@@ -14299,11 +14431,13 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
for (auto & o : overrides) {
if (o.key[0] == 0) break;
if (o.tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {
- gguf_set_val_f32(ctx_out, o.key, o.float_value);
+ gguf_set_val_f32(ctx_out, o.key, o.val_f64);
} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_INT) {
- gguf_set_val_i32(ctx_out, o.key, o.int_value);
+ gguf_set_val_i32(ctx_out, o.key, o.val_i64);
} else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {
- gguf_set_val_bool(ctx_out, o.key, o.bool_value);
+ gguf_set_val_bool(ctx_out, o.key, o.val_bool);
+ } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
+ gguf_set_val_str(ctx_out, o.key, o.val_str);
} else {
LLAMA_LOG_WARN("%s: unknown KV override type for key %s\n", __func__, o.key);
}
@@ -14622,7 +14756,7 @@ static int llama_apply_lora_from_file_internal(
std::unique_ptr<llama_model_loader> ml;
if (path_base_model) {
LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model);
- ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ nullptr));
+ ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*check_tensors*/ false, /*kv_overrides*/ nullptr));
ml->init_mappings(/*prefetch*/ false); // no prefetching
}
@@ -14881,12 +15015,13 @@ struct llama_model_params llama_model_default_params() {
/*.vocab_only =*/ false,
/*.use_mmap =*/ true,
/*.use_mlock =*/ false,
+ /*.check_tensors =*/ false,
};
- if (llamafile_has_metal()) {
- // // note: we usually have plenty of VRAM, so by default offload all layers to the GPU
- // result.n_gpu_layers = 999; // [jart] let llamafile/gpu.c handle this
- }
+#ifdef GGML_USE_METAL // [jart] let llamafile/gpu.c handle this
+ // note: we usually have plenty of VRAM, so by default offload all layers to the GPU
+ result.n_gpu_layers = 999;
+#endif
return result;
}
@@ -14917,6 +15052,7 @@ struct llama_context_params llama_context_default_params() {
/*.logits_all =*/ false,
/*.embeddings =*/ false,
/*.offload_kqv =*/ true,
+ /*.flash_attn =*/ false,
/*.abort_callback =*/ nullptr,
/*.abort_callback_data =*/ nullptr,
};
@@ -15078,6 +15214,7 @@ struct llama_context * llama_new_context_with_model(
cparams.defrag_thold = params.defrag_thold;
cparams.embeddings = params.embeddings;
cparams.offload_kqv = params.offload_kqv;
+ cparams.flash_attn = params.flash_attn;
cparams.pooling_type = params.pooling_type;
cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
@@ -15085,12 +15222,20 @@ struct llama_context * llama_new_context_with_model(
cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;
// this is necessary due to kv_self.n being padded later during inference
- cparams.n_ctx = GGML_PAD(cparams.n_ctx, 32);
+ cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);
// with causal attention, the batch size is limited by the context size
cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
- cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
+ // the batch has to be at least GGML_KQ_MASK_PAD because we will be padding the KQ_mask
+ // this is required by GPU kernels in order to avoid out-of-bounds accesses (e.g. ggml_flash_attn_ext)
+ // ref: https://github.com/ggerganov/llama.cpp/pull/5021
+ if (cparams.n_batch < GGML_KQ_MASK_PAD) {
+ LLAMA_LOG_WARN("%s: n_batch is less than GGML_KQ_MASK_PAD - increasing to %d\n", __func__, GGML_KQ_MASK_PAD);
+ cparams.n_batch = GGML_KQ_MASK_PAD;
+ }
+
+ cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
cparams.n_yarn_orig_ctx = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :
hparams.n_yarn_orig_ctx != 0 ? hparams.n_yarn_orig_ctx :
@@ -15122,6 +15267,23 @@ struct llama_context * llama_new_context_with_model(
}
}
+ if (cparams.flash_attn && hparams.use_alibi) {
+ LLAMA_LOG_WARN("%s: flash_attn is not yet compatible with ALiBi - forcing off\n", __func__);
+ cparams.flash_attn = false;
+ }
+
+ if (cparams.flash_attn && model->arch == LLM_ARCH_GROK) {
+ LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__);
+ cparams.flash_attn = false;
+ }
+
+// #ifdef GGML_USE_HIPBLAS [jart]
+ if (cparams.flash_attn && llamafile_has_amd_gpu()) {
+ LLAMA_LOG_WARN("%s: flash_attn is not yet compatible with AMD GPUs builds - forcing off\n", __func__);
+ cparams.flash_attn = false;
+ }
+// #endif
+
if (params.seed == LLAMA_DEFAULT_SEED) {
params.seed = _rand64();
}
@@ -15129,6 +15291,7 @@ struct llama_context * llama_new_context_with_model(
LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);
LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);
LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);
+ LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn);
LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);
LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);
@@ -15247,7 +15410,7 @@ if (llamafile_has_metal()) {
}
ctx->backends.push_back(ctx->backend_cpu);
- if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, kv_size, cparams.offload_kqv)) {
+ if (!llama_kv_cache_init(ctx->kv_self, ctx, type_k, type_v, kv_size, cparams.offload_kqv)) {
LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__);
llama_free(ctx);
return nullptr;
@@ -15848,6 +16011,7 @@ size_t llama_state_get_size(const struct llama_context * ctx) {
const size_t s_kv_head = sizeof(uint32_t);
const size_t s_kv_size = sizeof(uint32_t);
const size_t s_kv_used = sizeof(uint32_t);
+ const size_t s_v_trans = sizeof(uint32_t);
const size_t s_kv = ctx->kv_self.total_size();
const size_t s_kv_cell = sizeof(llama_pos) + sizeof(size_t) + cparams.n_seq_max*sizeof(llama_seq_id);
const size_t s_kv_cells = ctx->kv_self.size * s_kv_cell;
@@ -15865,10 +16029,14 @@ size_t llama_state_get_size(const struct llama_context * ctx) {
+ s_kv_head
+ s_kv_size
+ s_kv_used
+ + s_v_trans
+ s_kv
+ s_kv_cells
);
+ // on session change it is very likely that the state size has changed - so we need to update this function
+ static_assert(LLAMA_SESSION_VERSION == 6, "So you just bumped the session version - good. But did you remember to update llama_state_get_size?");
+
return s_total;
}
@@ -16014,11 +16182,13 @@ static void llama_state_get_data_internal(struct llama_context * ctx, llama_data
const uint32_t kv_size = kv_self.size;
const size_t kv_buf_size = kv_self.total_size() / (kv_size ? kv_size : 1) * kv_head;
const uint32_t kv_used = kv_self.used;
+ const uint32_t v_trans = kv_self.v_trans ? 1 : 0;
data_ctx->write(&kv_buf_size, sizeof(kv_buf_size));
data_ctx->write(&kv_head, sizeof(kv_head));
data_ctx->write(&kv_size, sizeof(kv_size));
data_ctx->write(&kv_used, sizeof(kv_used));
+ data_ctx->write(&v_trans, sizeof(v_trans));
if (kv_buf_size) {
const size_t pre_kv_buf_size = data_ctx->get_size_written();
@@ -16031,7 +16201,7 @@ static void llama_state_get_data_internal(struct llama_context * ctx, llama_data
ggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size());
data_ctx->write(tmp_buf.data(), tmp_buf.size());
- if (kv_self.recurrent) {
+ if (kv_self.recurrent || !kv_self.v_trans) {
// v is contiguous for recurrent models
// TODO: use other tensors for state models than k and v
const size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head);
@@ -16164,11 +16334,15 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {
uint32_t kv_head;
uint32_t kv_size;
uint32_t kv_used;
+ uint32_t v_trans;
memcpy(&kv_buf_size, inp, sizeof(kv_buf_size)); inp += sizeof(kv_buf_size);
memcpy(&kv_head, inp, sizeof(kv_head)); inp += sizeof(kv_head);
memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size);
memcpy(&kv_used, inp, sizeof(kv_used)); inp += sizeof(kv_used);
+ memcpy(&v_trans, inp, sizeof(v_trans)); inp += sizeof(v_trans);
+
+ GGML_ASSERT(kv_self.v_trans == (bool) v_trans); // incompatible V transposition
if (kv_self.size != kv_size) {
// the KV cache needs to be big enough to load all the KV cells from the saved state
@@ -16178,6 +16352,8 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {
__func__, kv_head, kv_size, kv_self.size);
}
+ llama_kv_cache_clear(ctx);
+
if (kv_buf_size) {
const size_t pre_kv_buf_size = inp - src;
@@ -16189,7 +16365,7 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {
ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size);
inp += k_size;
- if (kv_self.recurrent) {
+ if (kv_self.recurrent || !kv_self.v_trans) {
// v is contiguous for recurrent models
// TODO: use other tensors for state models than k and v
const size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head);
@@ -16211,8 +16387,6 @@ size_t llama_state_set_data(struct llama_context * ctx, const uint8_t * src) {
GGML_ASSERT(kv_buf_size == inp - src - pre_kv_buf_size);
}
- llama_kv_cache_clear(ctx);
-
ctx->kv_self.head = kv_head;
ctx->kv_self.used = kv_used;
@@ -16472,28 +16646,49 @@ static size_t llama_state_seq_get_data_internal(struct llama_context * ctx, llam
}
}
- // For the values, they are transposed, so we also need the element size and get the element ranges from each row
- const uint32_t kv_size = kv_self.size;
- for (int il = 0; il < (int)n_layer; ++il) {
- // Write value type
- const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
- data_ctx.write(&v_type_i, sizeof(v_type_i));
+ // TODO: simplify, reduce copy-paste
+ if (!kv_self.v_trans) {
+ for (int il = 0; il < (int)n_layer; ++il) {
+ // Write value type
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ data_ctx.write(&v_type_i, sizeof(v_type_i));
- // Write element size
- const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);
- data_ctx.write(&v_size_el, sizeof(v_size_el));
+ // Write row size of value
+ const size_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa);
+ data_ctx.write(&v_size_row, sizeof(v_size_row));
- // For each row, we get the element values of each cell
- for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
- // Read each range of cells of v_size_el length each into tmp_buf and write out
+ // Read each range of cells of v_size length each into tmp_buf and write out
for (const auto & range : cell_ranges) {
const size_t range_size = range.second - range.first;
- const size_t src_offset = (range.first + j * kv_size) * v_size_el;
- tmp_buf.resize(range_size * v_size_el);
- ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), src_offset, tmp_buf.size());
+ tmp_buf.resize(range_size * v_size_row);
+ ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), range.first * v_size_row, range_size * v_size_row);
data_ctx.write(tmp_buf.data(), tmp_buf.size());
}
}
+ } else {
+ // For the values, they are transposed, so we also need the element size and get the element ranges from each row
+ const uint32_t kv_size = kv_self.size;
+ for (int il = 0; il < (int)n_layer; ++il) {
+ // Write value type
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ data_ctx.write(&v_type_i, sizeof(v_type_i));
+
+ // Write element size
+ const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);
+ data_ctx.write(&v_size_el, sizeof(v_size_el));
+
+ // For each row, we get the element values of each cell
+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
+ // Read each range of cells of v_size_el length each into tmp_buf and write out
+ for (const auto & range : cell_ranges) {
+ const size_t range_size = range.second - range.first;
+ const size_t src_offset = (range.first + j * kv_size) * v_size_el;
+ tmp_buf.resize(range_size * v_size_el);
+ ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), src_offset, tmp_buf.size());
+ data_ctx.write(tmp_buf.data(), tmp_buf.size());
+ }
+ }
+ }
}
return data_ctx.get_size_written();
@@ -16618,41 +16813,75 @@ size_t llama_state_seq_set_data(struct llama_context * ctx, const uint8_t * src,
}
}
- // For each layer, read the values for each cell (transposed)
- for (int il = 0; il < (int)n_layer; ++il) {
- // Read type of value
- int32_t v_type_i_ref;
- memcpy(&v_type_i_ref, inp, sizeof(v_type_i_ref));
- inp += sizeof(v_type_i_ref);
- const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
- if (v_type_i != v_type_i_ref) {
- llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);
- LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);
- return 0;
- }
+ // TODO: simplify, reduce copy-paste
+ if (!kv_self.v_trans) {
+ for (int il = 0; il < (int)n_layer; ++il) {
+ // Read type of value
+ int32_t v_type_i_ref;
+ memcpy(&v_type_i_ref, inp, sizeof(v_type_i_ref));
+ inp += sizeof(v_type_i_ref);
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ if (v_type_i != v_type_i_ref) {
+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);
+ LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);
+ return 0;
+ }
- // Read element size of value
- size_t v_size_el_ref;
- memcpy(&v_size_el_ref, inp, sizeof(v_size_el_ref));
- inp += sizeof(v_size_el_ref);
- const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);
- if (v_size_el != v_size_el_ref) {
- llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);
- LLAMA_LOG_ERROR("%s: mismatched value element size (%zu != %zu, layer %d)\n", __func__, v_size_el, v_size_el_ref, il);
- return 0;
- }
+ // Read row size of value
+ size_t v_size_row_ref;
+ memcpy(&v_size_row_ref, inp, sizeof(v_size_row_ref));
+ inp += sizeof(v_size_row_ref);
+ const size_t v_size_row = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa);
+ if (v_size_row != v_size_row_ref) {
+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);
+ LLAMA_LOG_ERROR("%s: mismatched value row size (%zu != %zu, layer %d)\n", __func__, v_size_row, v_size_row_ref, il);
+ return 0;
+ }
- if (cell_count) {
- // For each row in the transposed matrix, read the values for the whole cell range
- for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
- const size_t dst_offset = (kv_head + j * kv_size) * v_size_el;
- ggml_backend_tensor_set(kv_self.v_l[il], inp, dst_offset, cell_count * v_size_el);
- inp += cell_count * v_size_el;
+ if (cell_count) {
+ // Read and set the values for the whole cell range
+ ggml_backend_tensor_set(kv_self.v_l[il], inp, kv_head * v_size_row, cell_count * v_size_row);
+ inp += cell_count * v_size_row;
+ }
+ }
+ } else {
+ // For each layer, read the values for each cell (transposed)
+ for (int il = 0; il < (int)n_layer; ++il) {
+ // Read type of value
+ int32_t v_type_i_ref;
+ memcpy(&v_type_i_ref, inp, sizeof(v_type_i_ref));
+ inp += sizeof(v_type_i_ref);
+ const int32_t v_type_i = (int32_t)kv_self.v_l[il]->type;
+ if (v_type_i != v_type_i_ref) {
+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);
+ LLAMA_LOG_ERROR("%s: mismatched value type (%d != %d, layer %d)\n", __func__, v_type_i, v_type_i_ref, il);
+ return 0;
+ }
+
+ // Read element size of value
+ size_t v_size_el_ref;
+ memcpy(&v_size_el_ref, inp, sizeof(v_size_el_ref));
+ inp += sizeof(v_size_el_ref);
+ const size_t v_size_el = ggml_type_size(kv_self.v_l[il]->type);
+ if (v_size_el != v_size_el_ref) {
+ llama_kv_cache_seq_rm(kv_self, dest_seq_id, -1, -1);
+ LLAMA_LOG_ERROR("%s: mismatched value element size (%zu != %zu, layer %d)\n", __func__, v_size_el, v_size_el_ref, il);
+ return 0;
+ }
+
+ if (cell_count) {
+ // For each row in the transposed matrix, read the values for the whole cell range
+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
+ const size_t dst_offset = (kv_head + j * kv_size) * v_size_el;
+ ggml_backend_tensor_set(kv_self.v_l[il], inp, dst_offset, cell_count * v_size_el);
+ inp += cell_count * v_size_el;
+ }
}
}
}
const size_t nread = inp - src;
+
return nread;
}
@@ -17449,9 +17678,9 @@ const char * llama_print_system_info(void) {
s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | ";
s += "MATMUL_INT8 = " + std::to_string(ggml_cpu_has_matmul_int8()) + " | ";
#ifdef GGML_USE_LLAMAFILE
- s += "LAMMAFILE = 1 | ";
+ s += "LLAMAFILE = 1 | ";
#else
- s += "LAMMAFILE = 0 | ";
+ s += "LLAMAFILE = 0 | ";
#endif
return s.c_str();
@@ -17495,7 +17724,7 @@ const std::vector<std::pair<std::string, struct ggml_tensor *>> & llama_internal
void llama_log_set(ggml_log_callback log_callback, void * user_data) {
g_state.log_callback = log_callback ? log_callback : llama_log_callback_default;
g_state.log_callback_user_data = user_data;
-#ifdef GGML_USE_METAL
+#ifdef GGML_USE_METAL // [jart] do not want
ggml_backend_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);
#endif
}
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index d49b8dd2..ff608594 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -42,7 +42,7 @@
#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'
#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
-#define LLAMA_SESSION_VERSION 5
+#define LLAMA_SESSION_VERSION 6
#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ
#define LLAMA_STATE_SEQ_VERSION 1
@@ -71,6 +71,18 @@ extern "C" {
LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
};
+ // pre-tokenization types
+ enum llama_vocab_pre_type {
+ LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,
+ LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,
+ LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,
+ LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,
+ LLAMA_VOCAB_PRE_TYPE_FALCON = 4,
+ LLAMA_VOCAB_PRE_TYPE_MPT = 5,
+ LLAMA_VOCAB_PRE_TYPE_STARCODER = 6,
+ LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,
+ };
+
// note: these values should be synchronized with ggml_rope
// TODO: maybe move this enum to ggml.h (ggml_rope_type)
enum llama_rope_type {
@@ -198,15 +210,19 @@ extern "C" {
LLAMA_KV_OVERRIDE_TYPE_INT,
LLAMA_KV_OVERRIDE_TYPE_FLOAT,
LLAMA_KV_OVERRIDE_TYPE_BOOL,
+ LLAMA_KV_OVERRIDE_TYPE_STR,
};
struct llama_model_kv_override {
- char key[128];
enum llama_model_kv_override_type tag;
+
+ char key[128];
+
union {
- int64_t int_value;
- double float_value;
- bool bool_value;
+ int64_t val_i64;
+ double val_f64;
+ bool val_bool;
+ char val_str[128];
};
};
@@ -235,9 +251,10 @@ extern "C" {
const struct llama_model_kv_override * kv_overrides;
// Keep the booleans together to avoid misalignment during copy-by-value.
- bool vocab_only; // only load the vocabulary, no weights
- bool use_mmap; // use mmap if possible
- bool use_mlock; // force system to keep model in RAM
+ bool vocab_only; // only load the vocabulary, no weights
+ bool use_mmap; // use mmap if possible
+ bool use_mlock; // force system to keep model in RAM
+ bool check_tensors; // validate model tensor data
};
struct llama_context_params {
@@ -273,6 +290,7 @@ extern "C" {
bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)
bool embeddings; // if true, extract embeddings (together with logits)
bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU
+ bool flash_attn; // whether to use flash attention
// Abort callback
// if it returns true, execution of llama_decode() will be aborted
@@ -528,7 +546,7 @@ extern "C" {
// Returns the number of used KV cells (i.e. have at least one sequence assigned to them)
LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx);
- // Clear the KV cache
+ // Clear the KV cache - both cell info is erased and KV data is zeroed
LLAMA_API void llama_kv_cache_clear(
struct llama_context * ctx);
diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cpp
index 766fd813..f7202ee9 100644
--- a/llama.cpp/llava/llava-cli.cpp
+++ b/llama.cpp/llava/llava-cli.cpp
@@ -117,7 +117,6 @@ struct llava_context {
};
struct llava_context * volatile g_ctx;
-
static void sigint_handler(int signo) {
if (signo == SIGINT) {
printf("\n");
@@ -127,11 +126,11 @@ static void sigint_handler(int signo) {
}
static void show_additional_info(int /*argc*/, char ** argv) {
- LOG_TEE("\n example usage: %s -m <llava-v1.5-7b/ggml-model-q5_k.gguf> --mmproj <llava-v1.5-7b/mmproj-model-f16.gguf> --image <path/to/an/image.jpg> [--temp 0.1] [-p \"describe the image in detail.\"]\n", argv[0]);
+ LOG_TEE("\n example usage: %s -m <llava-v1.5-7b/ggml-model-q5_k.gguf> --mmproj <llava-v1.5-7b/mmproj-model-f16.gguf> --image <path/to/an/image.jpg> --image <path/to/another/image.jpg> [--temp 0.1] [-p \"describe the image in detail.\"]\n", argv[0]);
LOG_TEE(" note: a lower temperature value like 0.1 is recommended for better quality.\n");
}
-static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_params * params) {
+static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_params * params, const std::string & fname) {
// load and preprocess the image
llava_image_embed * embed = NULL;
@@ -147,9 +146,9 @@ static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_para
}
params->prompt = remove_image_from_prompt(prompt);
} else {
- embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, params->image.c_str());
+ embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, fname.c_str());
if (!embed) {
- LOG_TEE("%s: is %s really an image file?\n", __func__, params->image.c_str());
+ fprintf(stderr, "%s: is %s really an image file?\n", __func__, fname.c_str());
return NULL;
}
}
@@ -221,17 +220,7 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_
printf("\n");
}
-
-static struct llava_context * llava_init(gpt_params * params) {
- const char * clip_path = params->mmproj.c_str();
-
- auto prompt = params->prompt;
- if (prompt.empty()) {
- prompt = "describe the image in detail.";
- }
-
- auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);
-
+static struct llama_model * llava_init(gpt_params * params) {
llama_backend_init();
llama_numa_init(params->numa);
@@ -242,6 +231,19 @@ static struct llava_context * llava_init(gpt_params * params) {
LOG_TEE("%s: error: unable to load model\n" , __func__);
return NULL;
}
+ return model;
+}
+
+static struct llava_context * llava_init_context(gpt_params * params, llama_model * model) {
+ const char * clip_path = params->mmproj.c_str();
+
+ auto prompt = params->prompt;
+ if (prompt.empty()) {
+ prompt = "describe the image in detail.";
+ }
+
+ auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);
+
llama_context_params ctx_params = llama_context_params_from_gpt_params(*params);
ctx_params.n_ctx = params->n_ctx < 2048 ? 2048 : params->n_ctx; // we need a longer context size to process image embeddings
@@ -272,14 +274,14 @@ static void llava_free(struct llava_context * ctx_llava) {
llama_backend_free();
}
-int llava_cli(int argc, char ** argv, gpt_params * params) {
+static void llama_log_callback_logTee(ggml_log_level level, const char * text, void * user_data) {
+ (void) level;
+ (void) user_data;
+ LOG_TEE("%s", text);
+}
- auto ctx_llava = llava_init(params);
- if (ctx_llava == NULL) {
- fprintf(stderr, "%s: error: failed to init llava\n", __func__);
- return 1;
- }
- g_ctx = ctx_llava;
+int llava_cli(int argc, char ** argv, gpt_params & params) {
+ ggml_time_init();
struct sigaction sa;
sa.sa_handler = sigint_handler;
@@ -287,17 +289,42 @@ int llava_cli(int argc, char ** argv, gpt_params * params) {
sa.sa_flags = 0;
sigaction(SIGINT, &sa, NULL);
- auto image_embed = load_image(ctx_llava, params);
- if (!image_embed) {
+#ifndef LOG_DISABLE_LOGS
+ log_set_target(log_filename_generator("llava", "log"));
+ LOG_TEE("Log start\n");
+ log_dump_cmdline(argc, argv);
+ llama_log_set(llama_log_callback_logTee, nullptr);
+#endif // LOG_DISABLE_LOGS
+
+ if (params.mmproj.empty() || (params.image.empty() && !prompt_contains_image(params.prompt))) {
+ gpt_print_usage(argc, argv, params);
+ show_additional_info(argc, argv);
+ return 1;
+ }
+ auto model = llava_init(&params);
+ if (model == NULL) {
+ fprintf(stderr, "%s: error: failed to init llava model\n", __func__);
return 1;
}
- // process the prompt
- process_prompt(ctx_llava, image_embed, params, params->prompt);
+ for (auto & image : params.image) {
+ auto ctx_llava = llava_init_context(&params, model);
- llama_print_timings(ctx_llava->ctx_llama);
+ auto image_embed = load_image(ctx_llava, &params, image);
+ if (!image_embed) {
+ std::cerr << "error: failed to load image " << image << ". Terminating\n\n";
+ return 1;
+ }
+
+ // process the prompt
+ process_prompt(ctx_llava, image_embed, &params, params.prompt);
+
+ llama_print_timings(ctx_llava->ctx_llama);
+ llava_image_embed_free(image_embed);
+ ctx_llava->model = NULL;
+ llava_free(ctx_llava);
+ }
+ llama_free_model(model);
- llava_image_embed_free(image_embed);
- llava_free(ctx_llava);
return 0;
}
diff --git a/llama.cpp/llava/llava.h b/llama.cpp/llava/llava.h
index 47b6784c..d6d1272f 100644
--- a/llama.cpp/llava/llava.h
+++ b/llama.cpp/llava/llava.h
@@ -46,7 +46,7 @@ LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed);
/** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */
LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past);
-LLAVA_API int llava_cli(int argc, char ** argv, gpt_params * params);
+LLAVA_API int llava_cli(int argc, char ** argv, gpt_params & params);
#ifdef __cplusplus
}
diff --git a/llama.cpp/log.h b/llama.cpp/log.h
index a7448024..3c2d077c 100644
--- a/llama.cpp/log.h
+++ b/llama.cpp/log.h
@@ -235,7 +235,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::
// INTERNAL, DO NOT USE
// USE LOG() INSTEAD
//
-#if !defined(_MSC_VER) or defined(__INTEL_LLVM_COMPILER)
+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER)
#define LOG_IMPL(str, ...) \
do { \
if (LOG_TARGET != nullptr) \
@@ -258,7 +258,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::
// INTERNAL, DO NOT USE
// USE LOG_TEE() INSTEAD
//
-#if !defined(_MSC_VER) or defined(__INTEL_LLVM_COMPILER)
+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER)
#define LOG_TEE_IMPL(str, ...) \
do { \
if (LOG_TARGET != nullptr) \
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 84388b1e..988438bf 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -211,7 +211,7 @@ int main(int argc, char ** argv) {
if (!params.mmproj.empty() &&
(!params.image.empty() ||
params.prompt.find("<img src=\"") != std::string::npos)) {
- return llava_cli(argc, argv, &params);
+ return llava_cli(argc, argv, params);
}
// TODO: Dump params ?
@@ -404,7 +404,7 @@ int main(int argc, char ** argv) {
log_tostr(embd_inp.empty()), n_matching_session_tokens, embd_inp.size(), session_tokens.size(), embd_inp.size());
// if we will use the cache for the full prompt without reaching the end of the cache, force
- // reevaluation of the last token token to recalculate the cached logits
+ // reevaluation of the last token to recalculate the cached logits
if (!embd_inp.empty() && n_matching_session_tokens == embd_inp.size() && session_tokens.size() > embd_inp.size()) {
LOGLN("recalculate the cached logits (do): session_tokens.resize( %zu )", embd_inp.size() - 1);
@@ -590,7 +590,6 @@ int main(int argc, char ** argv) {
}
struct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams);
- bool should_show_special_tokens = sparams.grammar.empty();
while ((n_remain != 0 && !is_antiprompt) || params.interactive) {
// predict
@@ -804,8 +803,7 @@ int main(int argc, char ** argv) {
// display text
if (input_echo && display) {
for (auto id : embd) {
- const std::string token_str =
- llama_token_to_piece(ctx, id, should_show_special_tokens);
+ const std::string token_str = llama_token_to_piece(ctx, id);
printf("%s", token_str.c_str());
if (embd.size() > 1) {
@@ -971,7 +969,7 @@ int main(int argc, char ** argv) {
for (size_t i = original_size; i < embd_inp.size(); ++i) {
const llama_token token = embd_inp[i];
output_tokens.push_back(token);
- output_ss << llama_token_to_piece(ctx, token, should_show_special_tokens);
+ output_ss << llama_token_to_piece(ctx, token);
}
n_remain -= line_inp.size();
diff --git a/llama.cpp/perplexity/perplexity.cpp b/llama.cpp/perplexity/perplexity.cpp
index e06e5071..d192b9de 100644
--- a/llama.cpp/perplexity/perplexity.cpp
+++ b/llama.cpp/perplexity/perplexity.cpp
@@ -218,17 +218,22 @@ static void process_logits(std::ostream& out, int n_vocab, const float * logits,
}
struct kl_divergence_result {
- double sum_nll = 0;
- double sum_nll2 = 0;
- double sum_kld = 0;
- double sum_kld2 = 0;
- double sum_nll_diff = 0;
- double sum_nll_diff2 = 0;
- size_t n_same_top = 0;
- size_t count = 0;
+ double sum_nll = 0.0;
+ double sum_nll2 = 0.0;
+ double sum_nll_base = 0.0;
+ double sum_nll_base2 = 0.0;
+ double sum_nll_nll_base = 0.0;
+ double sum_kld = 0.0;
+ double sum_kld2 = 0.0;
+ double sum_p_diff = 0.0;
+ double sum_p_diff2 = 0.0;
+ double sum_p_diff4 = 0.0;
+ float max_p_diff = 0.0f;
+ size_t n_same_top = 0.0;
+ size_t count = 0.0;
};
-static double log_softmax(int n_vocab, const float * logits, const uint16_t * base_log_prob, int tok, kl_divergence_result & kld) {
+static std::pair<double, float> log_softmax(int n_vocab, const float * logits, const uint16_t * base_log_prob, int tok, kl_divergence_result & kld) {
float max_logit = logits[0];
int imax = 0;
for (int i = 1; i < n_vocab; ++i) {
@@ -246,12 +251,17 @@ static double log_softmax(int n_vocab, const float * logits, const uint16_t * ba
const float scale = d[0];
const float min_log_prob = d[1];
base_log_prob += 4;
- float nll = max_logit + log_sum_exp - logits[tok];
+
+ const float nll = max_logit + log_sum_exp - logits[tok];
kld.sum_nll += nll;
kld.sum_nll2 += nll*nll;
- nll += (scale*base_log_prob[tok] + min_log_prob);
- kld.sum_nll_diff += nll;
- kld.sum_nll_diff2 += nll*nll;
+
+ const float nll_base = -(scale*base_log_prob[tok] + min_log_prob);
+ kld.sum_nll_base += nll_base;
+ kld.sum_nll_base2 += nll_base*nll_base;
+
+ kld.sum_nll_nll_base += nll*nll_base;
+
max_logit += log_sum_exp;
double sum = 0;
int imax_base = -1;
@@ -271,34 +281,50 @@ static double log_softmax(int n_vocab, const float * logits, const uint16_t * ba
kld.sum_kld2 += sum*sum;
++kld.count;
if (imax == imax_base) ++kld.n_same_top;
- return sum;
+
+ const float p_base = expf(-nll_base);
+ const float p = expf(-nll);
+ const float p_diff = p - p_base;
+ kld.sum_p_diff += p_diff;
+ const double p_diff2 = p_diff*p_diff;
+ kld.sum_p_diff2 += p_diff2;
+ kld.sum_p_diff4 += p_diff2*p_diff2;
+ kld.max_p_diff = std::max(kld.max_p_diff, std::fabs(p_diff));
+
+ return std::make_pair(sum, p_diff);
}
static void process_logits(int n_vocab, const float * logits, const int * tokens, int n_token,
std::vector<std::thread> & workers, const std::vector<uint16_t> & base_log_probs, kl_divergence_result & kld,
- float * kld_values) {
+ float * kld_values, float * p_diff_values) {
std::mutex mutex;
const int nv = 2*((n_vocab + 1)/2) + 4;
int counter = 0;
- auto compute = [&mutex, &counter, &base_log_probs, &kld, n_vocab, logits, tokens, n_token, nv, kld_values] () {
+ auto compute = [&mutex, &counter, &base_log_probs, &kld, n_vocab, logits, tokens, n_token, nv, kld_values, p_diff_values] () {
kl_divergence_result local_kld;
while (true) {
std::unique_lock<std::mutex> lock(mutex);
int i = counter++;
if (i >= n_token) {
- kld.sum_nll += local_kld.sum_nll;
- kld.sum_nll2 += local_kld.sum_nll2;
- kld.sum_kld += local_kld.sum_kld;
- kld.sum_kld2 += local_kld.sum_kld2;
- kld.sum_nll_diff += local_kld.sum_nll_diff;
- kld.sum_nll_diff2 += local_kld.sum_nll_diff2;
- kld.n_same_top += local_kld.n_same_top;
- kld.count += local_kld.count;
+ kld.sum_nll += local_kld.sum_nll;
+ kld.sum_nll2 += local_kld.sum_nll2;
+ kld.sum_nll_base += local_kld.sum_nll_base;
+ kld.sum_nll_base2 += local_kld.sum_nll_base2;
+ kld.sum_nll_nll_base += local_kld.sum_nll_nll_base;
+ kld.sum_kld += local_kld.sum_kld;
+ kld.sum_kld2 += local_kld.sum_kld2;
+ kld.sum_p_diff += local_kld.sum_p_diff;
+ kld.sum_p_diff2 += local_kld.sum_p_diff2;
+ kld.sum_p_diff4 += local_kld.sum_p_diff4;
+ kld.n_same_top += local_kld.n_same_top;
+ kld.max_p_diff = std::max(kld.max_p_diff, local_kld.max_p_diff);
+ kld.count += local_kld.count;
break;
}
lock.unlock();
- double v = log_softmax(n_vocab, logits + i*n_vocab, base_log_probs.data() + i*nv, tokens[i+1], local_kld);
- kld_values[i] = (float)v;
+ std::pair<double, float> v = log_softmax(n_vocab, logits + i*n_vocab, base_log_probs.data() + i*nv, tokens[i+1], local_kld);
+ kld_values[i] = (float)v.first;
+ p_diff_values[i] = v.second;
}
};
for (auto & w : workers) {
@@ -1713,7 +1739,8 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {
GGML_ASSERT(llama_add_eos_token(llama_get_model(ctx)) != 1);
std::vector<uint16_t> log_probs_uint16(size_t(n_ctx - 1 - n_ctx/2) * nv);
- std::vector<float> kld_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);
+ std::vector<float> kld_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);
+ std::vector<float> p_diff_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);
std::vector<float> logits;
if (num_batches > 1) {
logits.reserve(n_ctx * n_vocab);
@@ -1730,9 +1757,18 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {
df = df > 0 && count > 10 ? sqrt(df/(count-1)) : 0.;
return std::make_pair(f, df);
};
+ auto covariance = [] (double suma, double sumb, double sumab, size_t count) {
+ if (count < 10) {
+ return 0.0;
+ }
+ double var = sumab/count - (suma/count)*(sumb/count);
+ var /= count - 1;
+ return var;
+ };
kl_divergence_result kld;
- auto kld_ptr = kld_values.data();
+ auto kld_ptr = kld_values.data();
+ auto p_diff_ptr = p_diff_values.data();
for (int i = 0; i < n_chunk; ++i) {
const int start = i * n_ctx;
@@ -1787,24 +1823,42 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {
}
fprintf(stderr, "%.2f minutes\n", total_seconds / 60.0);
- printf("\nchunk PPL ln(PPL(Q)/PPL(base)) KL-Divergence Same top\n");
+ printf("\nchunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p\n");
}
const int first = n_ctx/2;
const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);
process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,
- workers, log_probs_uint16, kld, kld_ptr);
- kld_ptr += n_ctx - 1 - first;
+ workers, log_probs_uint16, kld, kld_ptr, p_diff_ptr);
+ p_diff_ptr += n_ctx - 1 - first;
+ kld_ptr += n_ctx - 1 - first;
+
+ printf("%4d", i+1);
+
+ auto log_ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);
+ const double ppl_val = exp(log_ppl.first);
+ const double ppl_unc = ppl_val * log_ppl.second; // ppl_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl.second ** 2 )
+ printf(" %9.4lf ± %9.4lf", ppl_val, ppl_unc);
+
+ auto log_ppl_base = mean_and_uncertainty(kld.sum_nll_base, kld.sum_nll_base2, kld.count);
+ const double log_ppl_cov = covariance(kld.sum_nll, kld.sum_nll_base, kld.sum_nll_nll_base, kld.count);
+ const double log_ppl_ratio_val = log_ppl.first - log_ppl_base.first;
+ const double log_ppl_ratio_unc = sqrt(log_ppl.second*log_ppl.second + log_ppl_base.second*log_ppl_base.second - 2.0*log_ppl_cov);
+ printf(" %10.5lf ± %10.5lf", log_ppl_ratio_val, log_ppl_ratio_unc);
+
+ auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);
+ printf(" %10.5lf ± %10.5lf", kl_div.first, kl_div.second);
- auto ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);
- auto log_ppl_ratio = mean_and_uncertainty(kld.sum_nll_diff, kld.sum_nll_diff2, kld.count);
- auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);
- auto p_top = 1.*kld.n_same_top/kld.count;
- auto d_p_top = sqrt(p_top*(1 - p_top)/(kld.count - 1));
+ auto p_diff_mse = mean_and_uncertainty(kld.sum_p_diff2, kld.sum_p_diff4, kld.count);
+ const double p_diff_rms_val = sqrt(p_diff_mse.first);
+ const double p_diff_rms_unc = 0.5/p_diff_rms_val * p_diff_mse.second;
+ printf(" %6.3lf ± %6.3lf %%", 100.0*p_diff_rms_val, 100.0*p_diff_rms_unc);
- printf("%4d %10.4lf %10.5lf ± %10.5f %10.5f ± %10.5lf %.5f ± %.5f\n", i+1, exp(ppl.first),
- log_ppl_ratio.first, log_ppl_ratio.second, kl_div.first, kl_div.second,
- p_top, d_p_top);
+ double p_top_val = 1.*kld.n_same_top/kld.count;
+ double p_top_unc = sqrt(p_top_val*(1 - p_top_val)/(kld.count - 1));
+ printf(" %6.3lf ± %6.3lf %%", 100.0*p_top_val, 100.0*p_top_unc);
+
+ printf("\n");
fflush(stdout);
@@ -1815,31 +1869,97 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {
if (kld.count < 100) return; // we do not wish to do statistics on so few values
std::sort(kld_values.begin(), kld_values.end());
+ std::sort(p_diff_values.begin(), p_diff_values.end());
+
+ printf("====== Perplexity statistics ======\n");
+
+ auto log_ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);
+ const double ppl_val = exp(log_ppl.first);
+ const double ppl_unc = ppl_val * log_ppl.second; // ppl_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl.second ** 2 )
+ printf("Mean PPL(Q) : %10.6lf ± %10.6lf\n", ppl_val, ppl_unc);
+
+ auto log_ppl_base = mean_and_uncertainty(kld.sum_nll_base, kld.sum_nll_base2, kld.count);
+ const double ppl_base_val = exp(log_ppl_base.first);
+ const double ppl_base_unc = ppl_base_val * log_ppl_base.second; // ppl_base_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl_base.second ** 2 )
+ printf("Mean PPL(base) : %10.6lf ± %10.6lf\n", ppl_base_val, ppl_base_unc);
+
+ const double log_ppl_cov = covariance(kld.sum_nll, kld.sum_nll_base, kld.sum_nll_nll_base, kld.count);
+ // printf("Cov(ln(PPL(Q)), ln(PPL(base))): %10.6lf\n", log_ppl_cov);
+ const double log_ppl_cor = log_ppl_cov / (log_ppl.second*log_ppl_base.second);
+ printf("Cor(ln(PPL(Q)), ln(PPL(base))): %6.2lf%%\n", 100.0*log_ppl_cor);
+
+ const double log_ppl_ratio_val = log_ppl.first - log_ppl_base.first;
+ const double log_ppl_ratio_unc = sqrt(log_ppl.second*log_ppl.second + log_ppl_base.second*log_ppl_base.second - 2.0*log_ppl_cov);
+ printf("Mean ln(PPL(Q)/PPL(base)) : %10.6lf ± %10.6lf\n", log_ppl_ratio_val, log_ppl_ratio_unc);
- printf("===== KL-divergence statistics\n");
+ const double ppl_ratio_val = exp(log_ppl_ratio_val);
+ const double ppl_ratio_unc = ppl_ratio_val * log_ppl_ratio_unc; // ppl_ratio_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl_ratio.second ** 2 )
+ printf("Mean PPL(Q)/PPL(base) : %10.6lf ± %10.6lf\n", ppl_ratio_val, ppl_ratio_unc);
+
+ const double ppl_cov = ppl_val * ppl_base_val * log_ppl_cov;
+ const double ppl_diff_val = ppl_val - ppl_base_val;
+ const double ppl_diff_unc = sqrt(ppl_unc*ppl_unc + ppl_base_unc*ppl_base_unc - 2.0*ppl_cov);
+ printf("Mean PPL(Q)-PPL(base) : %10.6lf ± %10.6lf\n", ppl_diff_val, ppl_diff_unc);
+
+ printf("\n");
+
+ printf("====== KL divergence statistics ======\n");
auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);
- printf("Average: %10.6f ±%10.6lf\n", kl_div.first, kl_div.second);
+ printf("Mean KLD: %10.6lf ± %10.6lf\n", kl_div.first, kl_div.second);
auto kld_median = kld_values.size()%2 == 0 ? 0.5f*(kld_values[kld_values.size()/2] + kld_values[kld_values.size()/2-1])
: kld_values[kld_values.size()/2];
- printf("Median : %10.6f\n", kld_median);
- auto percentile = [&kld_values] (float fraction) {
- if (fraction <= 0) return kld_values.front();
- if (fraction >= 1) return kld_values.back();
- float p = fraction*(kld_values.size() - 1);
+ auto percentile = [] (std::vector<float> values, float fraction) {
+ if (fraction <= 0) return values.front();
+ if (fraction >= 1) return values.back();
+ float p = fraction*(values.size() - 1);
size_t ip = size_t(p); p -= ip;
- return (1 - p)*kld_values[ip] + p*kld_values[std::min(ip+1, kld_values.size()-1)];
+ return (1 - p)*values[ip] + p*values[std::min(ip+1, values.size()-1)];
};
- printf("Maximum: %10.6f\n", kld_values.back());
- printf("KLD_99 : %10.6f\n", percentile(0.99f));
- printf("KLD_95 : %10.6f\n", percentile(0.95f));
- printf("KLD_90 : %10.6f\n", percentile(0.90f));
+ printf("Maximum KLD: %10.6f\n", kld_values.back());
+ printf("99.9%% KLD: %10.6f\n", percentile(kld_values, 0.999f));
+ printf("99.0%% KLD: %10.6f\n", percentile(kld_values, 0.990f));
+ printf("99.0%% KLD: %10.6f\n", percentile(kld_values, 0.990f));
+ printf("Median KLD: %10.6f\n", kld_median);
+ printf("10.0%% KLD: %10.6f\n", percentile(kld_values, 0.100f));
+ printf(" 5.0%% KLD: %10.6f\n", percentile(kld_values, 0.050f));
+ printf(" 1.0%% KLD: %10.6f\n", percentile(kld_values, 0.010f));
+ printf("Minimum KLD: %10.6f\n", kld_values.front());
+
+ printf("\n");
- printf("Minimum: %10.6f\n", kld_values.front());
- printf("KLD_01 : %10.6f\n", percentile(0.01f));
- printf("KLD_05 : %10.6f\n", percentile(0.05f));
- printf("KLD_10 : %10.6f\n", percentile(0.10f));
+ printf("====== Token probability statistics ======\n");
+
+ auto p_diff = mean_and_uncertainty(kld.sum_p_diff, kld.sum_p_diff2, kld.count);
+ printf("Mean Δp: %6.3lf ± %5.3lf %%\n", 100.0*p_diff.first, 100.0*p_diff.second);
+
+ auto p_diff_median = p_diff_values.size()%2 == 0 ? 0.5f*(p_diff_values[p_diff_values.size()/2] + p_diff_values[p_diff_values.size()/2-1])
+ : p_diff_values[p_diff_values.size()/2];
+
+ printf("Maximum Δp: %6.3lf%%\n", 100.0*p_diff_values.back());
+ printf("99.9%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.999f));
+ printf("99.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.990f));
+ printf("95.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.950f));
+ printf("90.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.900f));
+ printf("75.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.750f));
+ printf("Median Δp: %6.3lf%%\n", 100.0*p_diff_median);
+ printf("25.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.250f));
+ printf("10.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.100f));
+ printf(" 5.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.050f));
+ printf(" 1.0%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.010f));
+ printf(" 0.1%% Δp: %6.3lf%%\n", 100.0*percentile(p_diff_values, 0.001f));
+ printf("Minimum Δp: %6.3lf%%\n", 100.0*p_diff_values.front());
+
+ auto p_diff_mse = mean_and_uncertainty(kld.sum_p_diff2, kld.sum_p_diff4, kld.count);
+ // printf("MSE Δp : %10.6lf ± %10.6lf\n", p_diff_mse.first, p_diff_mse.second);
+
+ const double p_diff_rms_val = sqrt(p_diff_mse.first);
+ const double p_diff_rms_unc = 0.5/p_diff_rms_val * p_diff_mse.second;
+ printf("RMS Δp : %6.3lf ± %5.3lf %%\n", 100.0*p_diff_rms_val, 100.0*p_diff_rms_unc);
+
+ const double same_top_p = 1.0*kld.n_same_top/kld.count;
+ printf("Same top p: %6.3lf ± %5.3lf %%\n", 100.0*same_top_p, 100.0*sqrt(same_top_p*(1.0 - same_top_p)/(kld.count - 1)));
}
diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cpp
index d72b2719..27a9636e 100644
--- a/llama.cpp/quantize/quantize.cpp
+++ b/llama.cpp/quantize/quantize.cpp
@@ -59,6 +59,11 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {
{ "COPY", LLAMA_FTYPE_ALL_F32, "only copy tensors, no quantizing", },
};
+static const char * const LLM_KV_QUANTIZE_IMATRIX_FILE = "quantize.imatrix.file";
+static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrix.dataset";
+static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";
+static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";
+
static bool is_integer_str(const char *s) {
if (*s == '-') ++s;
if (!*s) return false;
@@ -122,7 +127,7 @@ static void usage(const char * executable) {
exit(1);
}
-static void load_imatrix(const std::string & imatrix_file, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
+static int load_imatrix(const std::string & imatrix_file, std::string & imatrix_dataset, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
std::ifstream in(imatrix_file.c_str(), std::ios::binary);
if (!in) {
printf("%s: failed to open %s\n",__func__, imatrix_file.c_str());
@@ -169,18 +174,33 @@ static void load_imatrix(const std::string & imatrix_file, std::unordered_map<st
printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n", __func__, int(e.size()), ncall, name.c_str());
}
}
- printf("%s: loaded %d importance matrix entries from %s\n", __func__, int(imatrix_data.size()), imatrix_file.c_str());
+
+ // latest imatrix version contains the dataset filename at the end of the file
+ int m_last_call = 0;
+ if (in.peek() != EOF) {
+ in.read((char *)&m_last_call, sizeof(m_last_call));
+ int dataset_len;
+ in.read((char *)&dataset_len, sizeof(dataset_len));
+ std::vector<char> dataset_as_vec(dataset_len);
+ in.read(dataset_as_vec.data(), dataset_len);
+ imatrix_dataset.assign(dataset_as_vec.begin(), dataset_as_vec.end());
+ printf("%s: imatrix dataset='%s'\n", __func__, imatrix_dataset.c_str());
+ }
+ printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_call);
+ return m_last_call;
}
-static void prepare_imatrix(const std::string & imatrix_file,
+static int prepare_imatrix(const std::string & imatrix_file,
+ std::string & imatrix_dataset,
const std::vector<std::string> & included_weights,
const std::vector<std::string> & excluded_weights,
std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
+ int m_last_call = -1;
if (!imatrix_file.empty()) {
- load_imatrix(imatrix_file, imatrix_data);
+ m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data);
}
if (imatrix_data.empty()) {
- return;
+ return m_last_call;
}
if (!excluded_weights.empty()) {
for (auto& name : excluded_weights) {
@@ -206,6 +226,7 @@ static void prepare_imatrix(const std::string & imatrix_file,
if (!imatrix_data.empty()) {
printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size()));
}
+ return m_last_call;
}
static ggml_type parse_ggml_type(const char * arg) {
@@ -220,43 +241,6 @@ static ggml_type parse_ggml_type(const char * arg) {
return result;
}
-static bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {
- const char* sep = strchr(data, '=');
- if (sep == nullptr || sep - data >= 128) {
- fprintf(stderr, "%s: malformed KV override '%s'\n", __func__, data);
- return false;
- }
- llama_model_kv_override kvo;
- std::strncpy(kvo.key, data, sep - data);
- kvo.key[sep - data] = 0;
- sep++;
- if (strncmp(sep, "int:", 4) == 0) {
- sep += 4;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
- kvo.int_value = std::atol(sep);
- } else if (strncmp(sep, "float:", 6) == 0) {
- sep += 6;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;
- kvo.float_value = std::atof(sep);
- } else if (strncmp(sep, "bool:", 5) == 0) {
- sep += 5;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;
- if (std::strcmp(sep, "true") == 0) {
- kvo.bool_value = true;
- } else if (std::strcmp(sep, "false") == 0) {
- kvo.bool_value = false;
- } else {
- fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data);
- return false;
- }
- } else {
- fprintf(stderr, "%s: invalid type for KV override '%s'\n", __func__, data);
- return false;
- }
- overrides.emplace_back(std::move(kvo));
- return true;
-}
-
int main(int argc, char ** argv) {
if (llamafile_has(argv, "--version")) {
@@ -340,10 +324,43 @@ int main(int argc, char ** argv) {
usage(argv[0]);
}
+ std::string imatrix_dataset;
std::unordered_map<std::string, std::vector<float>> imatrix_data;
- prepare_imatrix(imatrix_file, included_weights, excluded_weights, imatrix_data);
+ int m_last_call = prepare_imatrix(imatrix_file, imatrix_dataset, included_weights, excluded_weights, imatrix_data);
if (!imatrix_data.empty()) {
params.imatrix = &imatrix_data;
+ {
+ llama_model_kv_override kvo;
+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_FILE);
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;
+ strncpy(kvo.val_str, imatrix_file.c_str(), 127);
+ kvo.val_str[127] = '\0';
+ kv_overrides.emplace_back(std::move(kvo));
+ }
+ if (!imatrix_dataset.empty()) {
+ llama_model_kv_override kvo;
+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_DATASET);
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;
+ strncpy(kvo.val_str, imatrix_dataset.c_str(), 127);
+ kvo.val_str[127] = '\0';
+ kv_overrides.emplace_back(std::move(kvo));
+ }
+
+ {
+ llama_model_kv_override kvo;
+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES);
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
+ kvo.val_i64 = imatrix_data.size();
+ kv_overrides.emplace_back(std::move(kvo));
+ }
+
+ if (m_last_call > 0) {
+ llama_model_kv_override kvo;
+ std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS);
+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
+ kvo.val_i64 = m_last_call;
+ kv_overrides.emplace_back(std::move(kvo));
+ }
}
if (!kv_overrides.empty()) {
kv_overrides.emplace_back();
diff --git a/llama.cpp/quantize_row_q8_0.inc b/llama.cpp/quantize_row_q8_0.inc
deleted file mode 100644
index f4fe9d95..00000000
--- a/llama.cpp/quantize_row_q8_0.inc
+++ /dev/null
@@ -1,187 +0,0 @@
-static void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) {
- assert(QK8_0 == 32);
- assert(k % QK8_0 == 0);
- const int nb = k / QK8_0;
-
- block_q8_0 * restrict y = vy;
-
-#if defined(__ARM_NEON)
- for (int i = 0; i < nb; i++) {
- float32x4_t srcv [8];
- float32x4_t asrcv[8];
- float32x4_t amaxv[8];
-
- for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j);
- for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);
-
- for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);
- for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);
- for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);
-
- const float amax = vmaxvq_f32(amaxv[0]);
-
- const float d = amax / ((1 << 7) - 1);
- const float id = d ? 1.0f/d : 0.0f;
-
- y[i].d = GGML_FP32_TO_FP16(d);
-
- for (int j = 0; j < 8; j++) {
- const float32x4_t v = vmulq_n_f32(srcv[j], id);
- const int32x4_t vi = vcvtnq_s32_f32(v);
-
- y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);
- y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);
- y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);
- y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);
- }
- }
-#elif defined(__wasm_simd128__)
- for (int i = 0; i < nb; i++) {
- v128_t srcv [8];
- v128_t asrcv[8];
- v128_t amaxv[8];
-
- for (int j = 0; j < 8; j++) srcv[j] = wasm_v128_load(x + i*32 + 4*j);
- for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);
-
- for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);
- for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);
- for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);
-
- const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),
- wasm_f32x4_extract_lane(amaxv[0], 1)),
- MAX(wasm_f32x4_extract_lane(amaxv[0], 2),
- wasm_f32x4_extract_lane(amaxv[0], 3)));
-
- const float d = amax / ((1 << 7) - 1);
- const float id = d ? 1.0f/d : 0.0f;
-
- y[i].d = GGML_FP32_TO_FP16(d);
-
- for (int j = 0; j < 8; j++) {
- const v128_t v = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));
- const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);
-
- y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);
- y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);
- y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);
- y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);
- }
- }
-#elif defined(__AVX2__) || defined(__AVX__)
- for (int i = 0; i < nb; i++) {
- // Load elements into 4 AVX vectors
- __m256 v0 = _mm256_loadu_ps( x );
- __m256 v1 = _mm256_loadu_ps( x + 8 );
- __m256 v2 = _mm256_loadu_ps( x + 16 );
- __m256 v3 = _mm256_loadu_ps( x + 24 );
- x += 32;
-
- // Compute max(abs(e)) for the block
- const __m256 signBit = _mm256_set1_ps( -0.0f );
- __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );
- maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );
- maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );
- maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );
-
- __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );
- max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );
- max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );
- const float maxScalar = _mm_cvtss_f32( max4 );
-
- // Quantize these floats
- const float d = maxScalar / 127.f;
- y[i].d = GGML_FP32_TO_FP16(d);
- const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;
- const __m256 mul = _mm256_set1_ps( id );
-
- // Apply the multiplier
- v0 = _mm256_mul_ps( v0, mul );
- v1 = _mm256_mul_ps( v1, mul );
- v2 = _mm256_mul_ps( v2, mul );
- v3 = _mm256_mul_ps( v3, mul );
-
- // Round to nearest integer
- v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );
- v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );
- v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );
- v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );
-
- // Convert floats to integers
- __m256i i0 = _mm256_cvtps_epi32( v0 );
- __m256i i1 = _mm256_cvtps_epi32( v1 );
- __m256i i2 = _mm256_cvtps_epi32( v2 );
- __m256i i3 = _mm256_cvtps_epi32( v3 );
-
-#if defined(__AVX2__)
- // Convert int32 to int16
- i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15
- i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31
- // Convert int16 to int8
- i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31
-
- // We got our precious signed bytes, but the order is now wrong
- // These AVX2 pack instructions process 16-byte pieces independently
- // The following instruction is fixing the order
- const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );
- i0 = _mm256_permutevar8x32_epi32( i0, perm );
-
- _mm256_storeu_si256((__m256i *)y[i].qs, i0);
-#else
- // Since we don't have in AVX some necessary functions,
- // we split the registers in half and call AVX2 analogs from SSE
- __m128i ni0 = _mm256_castsi256_si128( i0 );
- __m128i ni1 = _mm256_extractf128_si256( i0, 1);
- __m128i ni2 = _mm256_castsi256_si128( i1 );
- __m128i ni3 = _mm256_extractf128_si256( i1, 1);
- __m128i ni4 = _mm256_castsi256_si128( i2 );
- __m128i ni5 = _mm256_extractf128_si256( i2, 1);
- __m128i ni6 = _mm256_castsi256_si128( i3 );
- __m128i ni7 = _mm256_extractf128_si256( i3, 1);
-
- // Convert int32 to int16
- ni0 = _mm_packs_epi32( ni0, ni1 );
- ni2 = _mm_packs_epi32( ni2, ni3 );
- ni4 = _mm_packs_epi32( ni4, ni5 );
- ni6 = _mm_packs_epi32( ni6, ni7 );
- // Convert int16 to int8
- ni0 = _mm_packs_epi16( ni0, ni2 );
- ni4 = _mm_packs_epi16( ni4, ni6 );
-
- _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
- _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
-#endif
- }
-#elif defined(__riscv_v_intrinsic)
-
- size_t vl = __riscv_vsetvl_e32m4(QK8_0);
-
- for (int i = 0; i < nb; i++) {
- // load elements
- vfloat32m4_t v_x = __riscv_vle32_v_f32m4(x+i*QK8_0, vl);
-
- vfloat32m4_t vfabs = __riscv_vfabs_v_f32m4(v_x, vl);
- vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl);
- vfloat32m1_t vmax = __riscv_vfredmax_vs_f32m4_f32m1(vfabs, tmp, vl);
- float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);
-
- const float d = amax / ((1 << 7) - 1);
- const float id = d ? 1.0f/d : 0.0f;
-
- y[i].d = GGML_FP32_TO_FP16(d);
-
- vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);
-
- // convert to integer
- vint16m2_t vi = __riscv_vfncvt_x_f_w_i16m2(x0, vl);
- vint8m1_t vs = __riscv_vncvt_x_x_w_i8m1(vi, vl);
-
- // store result
- __riscv_vse8_v_i8m1(y[i].qs , vs, vl);
- }
-#else
- GGML_UNUSED(nb);
- // scalar
- quantize_row_q8_0_reference(x, y, k);
-#endif
-}
diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cpp
index 447a6864..f42f5b92 100644
--- a/llama.cpp/sampling.cpp
+++ b/llama.cpp/sampling.cpp
@@ -71,7 +71,7 @@ void llama_sampling_reset(llama_sampling_context * ctx) {
void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed) {
if (seed == LLAMA_DEFAULT_SEED) {
- seed = time(NULL);
+ seed = std::random_device{}();
}
ctx->rng.seed(seed);
}
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index 441351d8..2d869576 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -2671,52 +2671,17 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
break;
}
sparams.chat_template = argv[i];
- }
- else if (arg == "--override-kv")
- {
+ } else if (arg == "--override-kv") {
if (++i >= argc) {
invalid_param = true;
break;
}
- char * sep = strchr(argv[i], '=');
- if (sep == nullptr || sep - argv[i] >= 128) {
- fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);
- invalid_param = true;
- break;
- }
- struct llama_model_kv_override kvo;
- std::strncpy(kvo.key, argv[i], sep - argv[i]);
- kvo.key[sep - argv[i]] = 0;
- sep++;
- if (strncmp(sep, "int:", 4) == 0) {
- sep += 4;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
- kvo.int_value = std::atol(sep);
- } else if (strncmp(sep, "float:", 6) == 0) {
- sep += 6;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;
- kvo.float_value = std::atof(sep);
- } else if (strncmp(sep, "bool:", 5) == 0) {
- sep += 5;
- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;
- if (std::strcmp(sep, "true") == 0) {
- kvo.bool_value = true;
- } else if (std::strcmp(sep, "false") == 0) {
- kvo.bool_value = false;
- } else {
- fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);
- invalid_param = true;
- break;
- }
- } else {
+ if (!parse_kv_override(argv[i], params.kv_overrides)) {
fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);
invalid_param = true;
break;
}
- params.kv_overrides.push_back(kvo);
- }
- else
- {
+ } else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
exit(1);
}
diff --git a/llama.cpp/unicode-data.cpp b/llama.cpp/unicode-data.cpp
index 22f8b0f0..e6bafb3a 100644
--- a/llama.cpp/unicode-data.cpp
+++ b/llama.cpp/unicode-data.cpp
@@ -1,4 +1,4 @@
-#include "unicode-data.h"
+#include "unicode-data.h"
#include <cstdint>
#include <map>
diff --git a/llama.cpp/unicode-data.h b/llama.cpp/unicode-data.h
index b99500b8..cb9dd8aa 100644
--- a/llama.cpp/unicode-data.h
+++ b/llama.cpp/unicode-data.h
@@ -12,5 +12,5 @@ extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_ma
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;
-extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
-extern const std::map<char32_t, char32_t> unicode_map_lowercase;
+extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
+extern const std::map<char32_t, char32_t> unicode_map_lowercase;
diff --git a/llama.cpp/unicode.cpp b/llama.cpp/unicode.cpp
index df8c5f58..f2ccda05 100644
--- a/llama.cpp/unicode.cpp
+++ b/llama.cpp/unicode.cpp
@@ -5,11 +5,14 @@
#include <cstddef>
#include <cstdint>
#include <map>
+#include <regex>
#include <stdexcept>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
+#include <locale>
+#include <codecvt>
static std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {
std::string result;
@@ -53,23 +56,22 @@ static uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset)
offset += 4;
return result;
}
- throw std::invalid_argument("invalid string");
+ throw std::invalid_argument("failed to convert utf8 to codepoint");
}
-static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {
- std::vector<uint16_t> result;
- if (/* 0x0000 <= cp && */ cp <= 0xffff) {
- result.emplace_back(cp);
- }
- else if (0x10000 <= cp && cp <= 0x10ffff) {
- result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));
- result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));
- }
- else {
- throw std::invalid_argument("invalid cpt");
- }
- return result;
-}
+//static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {
+// std::vector<uint16_t> result;
+// if (/* 0x0000 <= cp && */ cp <= 0xffff) {
+// result.emplace_back(cp);
+// return result;
+// }
+// if (0x10000 <= cp && cp <= 0x10ffff) {
+// result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));
+// result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));
+// return result;
+// }
+// throw std::invalid_argument("failed to convert codepoint to utf16");
+//}
//static std::vector<uint16_t> unicode_cpts_to_utf16(const std::vector<uint32_t> & cps) {
// std::vector<uint16_t> result;
@@ -80,28 +82,28 @@ static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {
// return result;
//}
-static uint32_t cpt_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {
- assert(offset < utf16.size());
- if (((utf16[0] >> 10) << 10) != 0xd800) {
- auto result = utf16[offset + 0];
- offset += 1;
- return result;
- }
-
- if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) {
- throw std::invalid_argument("invalid character");
- }
-
- auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));
- offset += 2;
- return result;
-}
+//static uint32_t unicode_cpt_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {
+// assert(offset < utf16.size());
+// if (((utf16[0] >> 10) << 10) != 0xd800) {
+// auto result = utf16[offset + 0];
+// offset += 1;
+// return result;
+// }
+//
+// if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) {
+// throw std::invalid_argument("invalid character");
+// }
+//
+// auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));
+// offset += 2;
+// return result;
+//}
//static std::vector<uint32_t> unicode_cpts_from_utf16(const std::vector<uint16_t> & utf16) {
// std::vector<uint32_t> result;
// size_t offset = 0;
// while (offset < utf16.size()) {
-// result.push_back(cpt_from_utf16(utf16, offset));
+// result.push_back(unicode_cpt_from_utf16(utf16, offset));
// }
// return result;
//}
@@ -194,34 +196,277 @@ static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
return map;
}
+static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
+ std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
+ return conv.from_bytes(s);
+}
+
+static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {
+ std::vector<std::string> bpe_encoded_words;
+ for (const auto & word : bpe_words) {
+ std::string text_utf;
+ auto utf_word = unicode_cpts_from_utf8(word);
+ for (size_t i = 0; i < utf_word.size(); ++i) {
+ text_utf += unicode_cpt_to_utf8(utf_word[i]);
+ }
+
+ std::string encoded_token;
+ for (char & c : text_utf) {
+ encoded_token += unicode_byte_to_utf8(c);
+ }
+ bpe_encoded_words.emplace_back(encoded_token);
+ }
+ return bpe_encoded_words;
+}
+
+// GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
+static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) {
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+
+ size_t start = 0;
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+
+ for (auto offset : offsets) {
+ std::string token;
+
+ bool collecting_numeric = false;
+ bool collecting_letter = false;
+ bool collecting_special = false;
+ bool collecting_whitespace_lookahead = false;
+ bool collecting = false;
+
+ std::vector<std::string> text_utf;
+ text_utf.reserve(offset);
+
+ for (size_t i = start; i < start + offset; ++i) {
+ text_utf.emplace_back(unicode_cpt_to_utf8(cpts[i]));
+ }
+
+ for (int i = 0; i < (int)text_utf.size(); i++) {
+ const std::string & utf_char = text_utf[i];
+ bool split_condition = false;
+ int bytes_remain = text_utf.size() - i;
+
+ // forward backward lookups
+ const std::string & utf_char_next = (i + 1 < (int)text_utf.size()) ? text_utf[i + 1] : "";
+ const std::string & utf_char_next_next = (i + 2 < (int)text_utf.size()) ? text_utf[i + 2] : "";
+
+ // handling contractions
+ if (!split_condition && bytes_remain >= 2) {
+ // 's|'t|'m|'d
+ if (utf_char == "\'" && (utf_char_next == "s" || utf_char_next == "t" || utf_char_next == "m" || utf_char_next == "d")) {
+ split_condition = true;
+ }
+ if (split_condition) {
+ if (token.size()) {
+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());
+ }
+ token = utf_char + utf_char_next;
+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());
+ token = "";
+ i++;
+ continue;
+ }
+ }
+ if (!split_condition && bytes_remain >= 3) {
+ // 're|'ve|'ll
+ if (utf_char == "\'" && (
+ (utf_char_next == "r" && utf_char_next_next == "e") ||
+ (utf_char_next == "v" && utf_char_next_next == "e") ||
+ (utf_char_next == "l" && utf_char_next_next == "l"))
+ ) {
+ split_condition = true;
+ }
+ if (split_condition) {
+ // current token + next token can be defined
+ if (token.size()) {
+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());
+ }
+ token = utf_char;
+ token += utf_char_next;
+ token += utf_char_next_next;
+
+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());
+ token = "";
+ i += 2;
+ continue;
+ }
+ }
+
+ if (!split_condition && !collecting) {
+ if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) {
+ collecting_letter = true;
+ collecting = true;
+ }
+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {
+ collecting_numeric = true;
+ collecting = true;
+ }
+ else if (
+ ((unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (unicode_cpt_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) ||
+ (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE)
+ ) {
+ collecting_special = true;
+ collecting = true;
+ }
+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) {
+ collecting_whitespace_lookahead = true;
+ collecting = true;
+ }
+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) {
+ split_condition = true;
+ }
+ }
+ else if (!split_condition && collecting) {
+ if (collecting_letter && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER) {
+ split_condition = true;
+ }
+ else if (collecting_numeric && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) {
+ split_condition = true;
+ }
+ else if (collecting_special && (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) {
+ split_condition = true;
+ }
+ else if (collecting_whitespace_lookahead && (unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {
+ split_condition = true;
+ }
+ }
+
+ if (utf_char_next == "") {
+ split_condition = true; // final
+ token += utf_char;
+ }
+
+ if (split_condition) {
+ if (token.size()) {
+ bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());
+ }
+ token = utf_char;
+ collecting = false;
+ collecting_letter = false;
+ collecting_numeric = false;
+ collecting_special = false;
+ collecting_whitespace_lookahead = false;
+ }
+ else {
+ token += utf_char;
+ }
+ }
+
+ start += offset;
+ }
+
+ return bpe_offsets;
+}
+
+// use std::wregex to split the text
+static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, const std::wstring & regex_expr, const std::vector<size_t> & offsets) {
+ std::wregex expr(regex_expr);
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+ size_t start = 0;
+ for (auto offset : offsets) {
+ std::wcregex_iterator it(wtext.data() + start, wtext.data() + start + offset, expr);
+ std::wcregex_iterator end;
+
+ int64_t start_idx = 0;
+ while (it != end) {
+ std::wcmatch match = *it;
+ if (match.position() > start_idx) {
+ bpe_offsets.emplace_back(match.position() - start_idx);
+ }
+ bpe_offsets.emplace_back(match.length());
+ start_idx = match.position() + match.length();
+ ++it;
+ }
+
+ if (start_idx < (int64_t) offset) {
+ bpe_offsets.emplace_back(offset - start_idx);
+ }
+ start += offset;
+ }
+
+ return bpe_offsets;
+}
+
+// use std::regex to split the text
+static std::vector<size_t> unicode_regex_split_stl(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
+ std::regex expr(regex_expr);
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+ size_t start = 0;
+ for (auto offset : offsets) {
+ std::cregex_iterator it(text.data() + start, text.data() + start + offset, expr);
+ std::cregex_iterator end;
+
+ int64_t start_idx = 0;
+ while (it != end) {
+ std::cmatch match = *it;
+ if (match.position() > start_idx) {
+ bpe_offsets.emplace_back(match.position() - start_idx);
+ }
+ bpe_offsets.emplace_back(match.length());
+ start_idx = match.position() + match.length();
+ ++it;
+ }
+
+ if (start_idx < (int64_t) offset) {
+ bpe_offsets.emplace_back(offset - start_idx);
+ }
+ start += offset;
+ }
+
+ return bpe_offsets;
+}
+
+static std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
+ std::vector<size_t> bpe_offsets;
+
+ (void)(text);
+ (void)(regex_expr);
+ (void)(offsets);
+ // TODO: this implementation is actually wrong, uncomment and run:
+ // make -j && ./bin/test-tokenizer-0 ../models/ggml-vocab-gpt-2.gguf
+ //if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") {
+ // bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);
+ //}
+
+ return bpe_offsets;
+}
+
//
// interface
//
std::string unicode_cpt_to_utf8(uint32_t cp) {
std::string result;
+
if (/* 0x00 <= cp && */ cp <= 0x7f) {
result.push_back(cp);
+ return result;
}
- else if (0x80 <= cp && cp <= 0x7ff) {
+ if (0x80 <= cp && cp <= 0x7ff) {
result.push_back(0xc0 | ((cp >> 6) & 0x1f));
result.push_back(0x80 | (cp & 0x3f));
+ return result;
}
- else if (0x800 <= cp && cp <= 0xffff) {
+ if (0x800 <= cp && cp <= 0xffff) {
result.push_back(0xe0 | ((cp >> 12) & 0x0f));
result.push_back(0x80 | ((cp >> 6) & 0x3f));
result.push_back(0x80 | (cp & 0x3f));
+ return result;
}
- else if (0x10000 <= cp && cp <= 0x10ffff) {
+ if (0x10000 <= cp && cp <= 0x10ffff) {
result.push_back(0xf0 | ((cp >> 18) & 0x07));
result.push_back(0x80 | ((cp >> 12) & 0x3f));
result.push_back(0x80 | ((cp >> 6) & 0x3f));
result.push_back(0x80 | (cp & 0x3f));
+ return result;
}
- else {
- throw std::invalid_argument("invalid codepoint");
- }
- return result;
+
+ throw std::invalid_argument("invalid codepoint");
}
std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {
@@ -275,3 +520,167 @@ char32_t unicode_tolower(char32_t cp) {
auto it = unicode_map_lowercase.find(cp);
return it == unicode_map_lowercase.end() ? cp : it->second;
}
+
+std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {
+ // unicode categories
+ static const std::map<std::string, int> k_ucat_enum = {
+ { "\\p{N}", CODEPOINT_TYPE_DIGIT },
+ { "\\p{L}", CODEPOINT_TYPE_LETTER },
+ { "\\p{P}", CODEPOINT_TYPE_PUNCTUATION },
+ };
+
+ static const std::map<int, int> k_ucat_cpt = {
+ { CODEPOINT_TYPE_DIGIT, 0xD1 },
+ { CODEPOINT_TYPE_LETTER, 0xD2 },
+ { CODEPOINT_TYPE_PUNCTUATION, 0xD3 },
+ };
+
+ static const std::map<int, std::string> k_ucat_map = {
+ { CODEPOINT_TYPE_DIGIT, "\x30-\x39" }, // 0-9
+ { CODEPOINT_TYPE_LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z
+ { CODEPOINT_TYPE_PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}
+ };
+
+ // compute collapsed codepoints only if needed by at least one regex
+ bool need_collapse = false;
+ for (auto & regex_expr : regex_exprs) {
+ // search for unicode categories
+ for (const auto & ucat : k_ucat_enum) {
+ if (std::string::npos != regex_expr.find(ucat.first)) {
+ need_collapse = true;
+ break;
+ }
+ }
+ }
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+
+ // generate a "collapsed" representation of the text, where all codepoints are replaced by a single byte
+ // ref: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2081479935
+ std::string text_collapsed;
+ if (need_collapse) {
+ // collapse all unicode categories
+ text_collapsed.resize(cpts.size());
+
+ for (size_t i = 0; i < cpts.size(); ++i) {
+ // keep single-byte codepoints as is
+ if (cpts[i] < 128) {
+ text_collapsed[i] = cpts[i];
+ continue;
+ }
+
+ const int cpt_type = unicode_cpt_type(cpts[i]);
+
+ if (k_ucat_cpt.find(cpt_type) != k_ucat_cpt.end()) {
+ text_collapsed[i] = k_ucat_cpt.at(cpt_type);
+ } else {
+ text_collapsed[i] = (char) 0xD0; // fallback
+ }
+ }
+ }
+
+ std::vector<size_t> bpe_offsets = { cpts.size() };
+
+ for (auto & regex_expr : regex_exprs) {
+ // first, see if we have an efficient custom regex implementation
+ auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets);
+
+ if (!tmp.empty()) {
+ bpe_offsets = std::move(tmp);
+ continue;
+ }
+
+ // fallback to general-purpose std::regex / std::wregex
+ try {
+ // if a unicode category is used in the regex, we use the collapsed text and replace the unicode category
+ // with the corresponding collapsed representation
+ bool use_collapsed = false;
+ for (auto & ucat : k_ucat_enum) {
+ if (std::string::npos != regex_expr.find(ucat.first)) {
+ use_collapsed = true;
+ break;
+ }
+ }
+
+ if (use_collapsed) {
+ // sanity-check that the original regex does not contain any non-ASCII characters
+ const auto cpts_regex = unicode_cpts_from_utf8(regex_expr);
+ for (size_t i = 0; i < cpts_regex.size(); ++i) {
+ if (cpts_regex[i] >= 128) {
+ throw std::runtime_error("Regex includes both unicode categories and non-ASCII characters - not supported");
+ }
+ }
+
+ // generate a collapsed representation of the regex
+ std::string regex_expr_collapsed;
+
+ // track if we are inside [], because nested [] are not allowed
+ bool inside = false;
+ for (size_t i = 0; i < regex_expr.size(); ++i) {
+ if (regex_expr[i] == '[' && (i == 0 || regex_expr[i - 1] != '\\')) {
+ regex_expr_collapsed += '[';
+ inside = true;
+ continue;
+ }
+
+ if (inside && regex_expr[i] == ']' && regex_expr[i - 1] != '\\') {
+ regex_expr_collapsed += ']';
+ inside = false;
+ continue;
+ }
+
+ if (regex_expr[i + 0] == '\\' && i + 4 < regex_expr.size() &&
+ regex_expr[i + 1] == 'p' &&
+ regex_expr[i + 2] == '{' &&
+ regex_expr[i + 4] == '}') {
+ const std::string pat = regex_expr.substr(i, 5);
+ if (k_ucat_enum.find(pat) != k_ucat_enum.end()) {
+ if (!inside) {
+ regex_expr_collapsed += '[';
+ }
+ regex_expr_collapsed += k_ucat_cpt.at(k_ucat_enum.at(pat));
+ regex_expr_collapsed += k_ucat_map.at(k_ucat_enum.at(pat));
+ if (!inside) {
+ regex_expr_collapsed += ']';
+ }
+ i += 4;
+ continue;
+ }
+ }
+
+ regex_expr_collapsed += regex_expr[i];
+ }
+
+ //printf("text_collapsed: %s\n", text_collapsed.c_str());
+ //printf("regex_expr_collapsed: %s\n", regex_expr_collapsed.c_str());
+ bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets);
+ } else {
+ // no unicode category used, we can use std::wregex directly
+ const std::wstring wtext = unicode_wstring_from_utf8(text);
+ const std::wstring wregex_expr = unicode_wstring_from_utf8(regex_expr);
+
+ //printf("text: %s\n", text.c_str());
+ //printf("regex_expr: %s\n", regex_expr.c_str());
+ bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets);
+ }
+ } catch (std::regex_error & e) {
+ fprintf(stderr, "Failed to process regex: '%s'\n", regex_expr.c_str());
+ fprintf(stderr, "Regex error: %s\n", e.what());
+ throw std::runtime_error("Failed to process regex");
+ }
+ }
+
+ std::vector<std::string> bpe_words;
+ bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size
+
+ size_t start = 0;
+ for (size_t & offset : bpe_offsets) {
+ bpe_words.emplace_back();
+ for (size_t i = start; i < start + offset; ++i) {
+ bpe_words.back() += unicode_cpt_to_utf8(cpts[i]);
+ }
+ start += offset;
+ }
+
+ return unicode_byte_encoding_process(bpe_words);
+}
diff --git a/llama.cpp/unicode.h b/llama.cpp/unicode.h
index 6a0be393..ce2bcef5 100644
--- a/llama.cpp/unicode.h
+++ b/llama.cpp/unicode.h
@@ -24,5 +24,6 @@ int unicode_cpt_type(const std::string & utf8);
std::string unicode_byte_to_utf8(uint8_t byte);
uint8_t unicode_utf8_to_byte(const std::string & utf8);
-// simple tolower that only implements one-to-one mapping, not one-to-many
char32_t unicode_tolower(char32_t cp);
+
+std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);
diff --git a/llamafile/cuda.c b/llamafile/cuda.c
index f0e64ec4..352f2ee4 100644
--- a/llamafile/cuda.c
+++ b/llamafile/cuda.c
@@ -89,6 +89,7 @@ GGML_CALL int ggml_backend_cuda_reg_devices(void);
static struct Cuda {
bool supported;
+ bool has_amd_gpu;
atomic_uint once;
typeof(ggml_cuda_link) *GGML_CALL link;
typeof(ggml_backend_cuda_buffer_type) *GGML_CALL buffer_type;
@@ -217,11 +218,12 @@ struct StringList {
size_t length;
};
-static int StringCompare(const void* a, const void* b) {
- return strcmp(*(const char**)a, *(const char**)b);
+static int StringCompare(const void *a, const void *b) {
+ return strcmp(*(const char **)a, *(const char **)b);
}
-static void AddStringToStringList(struct StringList *string_list, char *string, size_t string_length) {
+static void AddStringToStringList(struct StringList *string_list, char *string,
+ size_t string_length) {
struct StringListEntry *new_entry = malloc(sizeof(struct StringListEntry));
new_entry->string = malloc(sizeof(char) * (string_length + 1));
strncpy(new_entry->string, string, string_length);
@@ -376,7 +378,7 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {
char **names = NULL;
CopyStringListToStringArray(&name_list, &names);
- qsort(names, name_list.length, sizeof(char*), StringCompare);
+ qsort(names, name_list.length, sizeof(char *), StringCompare);
int num_names = RemoveDuplicatesFromSortedStringArray(names, name_list.length);
@@ -808,6 +810,7 @@ static bool import_cuda_impl(void) {
if (FLAG_nocompile) {
if ((FileExists(dso) || extract_cuda_dso(dso, "ggml-rocm")) &&
link_cuda_dso(dso, library_path)) {
+ ggml_cuda.has_amd_gpu = true;
return true;
} else {
goto TryNvidia;
@@ -820,10 +823,12 @@ static bool import_cuda_impl(void) {
case -1:
return false;
case false:
- if (link_cuda_dso(dso, library_path))
+ if (link_cuda_dso(dso, library_path)) {
+ ggml_cuda.has_amd_gpu = true;
return true;
- else
+ } else {
goto TryNvidia;
+ }
case true:
break;
default:
@@ -834,10 +839,12 @@ static bool import_cuda_impl(void) {
// Try building CUDA with ROCm SDK.
if (compiler_path) {
if (compile_amd(compiler_path, dso, src)) {
- if (link_cuda_dso(dso, library_path))
+ if (link_cuda_dso(dso, library_path)) {
+ ggml_cuda.has_amd_gpu = true;
return true;
- else
+ } else {
goto TryNvidia;
+ }
}
} else {
tinylog(__func__,
@@ -848,10 +855,12 @@ static bool import_cuda_impl(void) {
// Try extracting prebuilt tinyBLAS DSO from PKZIP.
if (extract_cuda_dso(dso, "ggml-rocm")) {
- if (link_cuda_dso(dso, library_path))
+ if (link_cuda_dso(dso, library_path)) {
+ ggml_cuda.has_amd_gpu = true;
return true;
- else
+ } else {
goto TryNvidia;
+ }
}
break;
@@ -934,6 +943,11 @@ bool llamafile_has_cuda(void) {
return ggml_cuda.supported;
}
+bool llamafile_has_amd_gpu(void) {
+ cosmo_once(&ggml_cuda.once, import_cuda);
+ return ggml_cuda.has_amd_gpu;
+}
+
GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {
if (!llamafile_has_cuda())
return 0;
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 514b7151..0503676c 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -50,11 +50,12 @@ bool llamafile_has_gpu(void);
int llamafile_gpu_layers(int);
bool llamafile_has_cuda(void);
bool llamafile_has_metal(void);
+bool llamafile_has_amd_gpu(void);
int llamafile_gpu_parse(const char *);
const char *llamafile_describe_gpu(void);
bool llamafile_sgemm(long, long, long, const void *, long, const void *, long, void *, long, int,
- int, int, int, int, int);
+ int, int, int, int, int, int);
struct ggml_tensor;
struct ggml_compute_params;
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 6d52ebb0..172bd41e 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -111,12 +111,14 @@ static const struct GemmFuncs {
* @param Atype is GGML data type of `A`
* @param Btype is GGML data type of `B`
* @param Ctype is GGML data type of `C`
+ * @param precision may be used to control the internal compute type
* @return true if this function was able to service the matmul request
*/
bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,
- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,
- int Ctype) {
- return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype);
+ void *C, long ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype,
+ int precision) {
+ return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype,
+ precision);
}
/**
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
index f734e9b3..9201cef6 100644
--- a/llamafile/sgemm.h
+++ b/llamafile/sgemm.h
@@ -8,23 +8,23 @@ struct ggml_tensor;
struct ggml_compute_params;
bool llamafile_sgemm_unsupported(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int, int, int);
bool llamafile_sgemm_amd_avx(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int, int, int);
bool llamafile_sgemm_amd_fma(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int, int, int);
bool llamafile_sgemm_amd_avx2(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int, int, int);
bool llamafile_sgemm_amd_avxvnni(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int, int, int);
bool llamafile_sgemm_amd_avx512f(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int, int, int);
bool llamafile_sgemm_amd_zen4(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int, int, int);
bool llamafile_sgemm_arm80(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int, int, int);
bool llamafile_sgemm_arm82(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int, int, int);
bool llamafile_mixmul_unsupported(struct ggml_compute_params *, const struct ggml_tensor *,
const struct ggml_tensor *, const struct ggml_tensor *,
diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cpp
index 3825989c..b7e49462 100644
--- a/llamafile/sgemm_sss_test.cpp
+++ b/llamafile/sgemm_sss_test.cpp
@@ -47,7 +47,7 @@ int main(int argc, char *argv[]) {
BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
for (int i = 0; i < m; ++i)
for (int j = 0; j < n; ++j) {
diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cpp
index 283973ad..651e230c 100644
--- a/llamafile/sgemm_vecdot_test.cpp
+++ b/llamafile/sgemm_vecdot_test.cpp
@@ -46,7 +46,7 @@ int main(int argc, char *argv[]) {
BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
double err_sum = 0;
long long err_worst = 0;
diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.inc
index 7e751699..1e29bada 100644
--- a/llamafile/tinyblas_cpu.inc
+++ b/llamafile/tinyblas_cpu.inc
@@ -383,12 +383,22 @@ inline __m512bh load(const float *p) {
#endif // __AVX512BF16__
////////////////////////////////////////////////////////////////////////////////////////////////////
-// FLOATING POINT MATRIX MULTIPLICATION
+// FLOATING POINT OUTPUT STREAMING
+
+inline void store(float *p, float f) {
+ *p = f;
+}
+
+inline void store(ggml_fp16_t *p, float f) {
+ *p = GGML_FP32_TO_FP16(f);
+}
-float id(float x) {
- return x;
+inline void store(ggml_bf16_t *p, float f) {
+ *p = GGML_FP32_TO_BF16(f);
}
-float (*veil)(float) = id;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// FLOATING POINT MATRIX MULTIPLICATION
template <int CONFIG, int KN, typename D, typename V, typename TA, typename TB, typename TC>
class tinyBLAS {
@@ -579,7 +589,7 @@ class tinyBLAS {
Cv[j][i]);
for (int j = 0; j < RN; ++j)
for (int i = 0; i < RM; ++i)
- *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]); // + logf(veil(-1));
+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));
}
}
@@ -700,7 +710,7 @@ class tinyBLAS_Q0_ARM {
}
for (int j = 0; j < RN; ++j)
for (int i = 0; i < RM; ++i)
- *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);
+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));
}
}
@@ -888,7 +898,7 @@ class tinyBLAS_Q0_AVX2 {
}
for (int j = 0; j < RN; ++j)
for (int i = 0; i < RM; ++i)
- *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);
+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));
}
}
@@ -926,52 +936,10 @@ class tinyBLAS_Q0_AVX2 {
};
#endif // __AVX2__
-} // namespace
-
-/**
- * Performs optimized matrix multiplication on CPU.
- *
- * This subroutine may compute C = Aᵀ * B with column major ordering.
- * Despite its name, this isn't a generalized implementation. Work is
- * only performed when a handwritten kernel is written and available.
- * Otherwise the caller should fall back to a general matmul routine.
- *
- * For example, for single-threaded single-precision GEMM you can say
- *
- * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1,
- * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);
- *
- * @param m is rows in `A` and `C`
- * @param n is cols in `B` and `C`
- * @param k is cols in `A` and rows in `B`
- * @param A is first input matrix (always transposed)
- * @param lda is row stride of `A`
- * @param B is second input matrix (never transposed)
- * @param ldb is row stride of `B`
- * @param C is input/output array of output matrices
- * @param ldc is row stride of `C`
- * @param ith is thread id (must be less than `nth`)
- * @param nth is number of threads (must be greater than zero)
- * @param Atype is GGML data type of `A`
- * @param Btype is GGML data type of `B`
- * @param Ctype is GGML data type of `C`
- * @return true if this function was able to service the matmul request
- */
-bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,
- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,
- int Ctype) {
-
- assert(m >= 0);
- assert(n >= 0);
- assert(k >= 0);
- assert(lda >= k);
- assert(ldb >= k);
- assert(ldc >= m);
- assert(nth > 0);
- assert(ith < nth);
-
- if (Ctype != GGML_TYPE_F32)
- return false;
+template <typename TC>
+bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const void *B, long ldb,
+ TC *C, long ldc, int ith, int nth, int task, int Atype, int Btype,
+ int precision) {
switch (Atype) {
@@ -981,22 +949,22 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
#if defined(__AVX512F__)
if (k % 16)
return false;
- tinyBLAS<0, 16, __m512, __m512, float, float, float> tb{
- k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 16, __m512, __m512, float, float, TC> tb{
+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX__) || defined(__AVX2__)
if (k % 8)
return false;
- tinyBLAS<0, 8, __m256, __m256, float, float, float> tb{
- k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 8, __m256, __m256, float, float, TC> tb{
+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_NEON)
if (k % 4)
return false;
- tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, float> tb{
- k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, TC> tb{
+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
@@ -1009,22 +977,22 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
if (k % 32)
return false;
if (Btype == GGML_TYPE_F32 && n < 2) {
- tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, float> tb{
- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, TC> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
}
if (Btype != GGML_TYPE_BF16)
return false;
- tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{
- k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC> tb{
+ k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX512F__)
if (k % 16)
return false;
- tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, float> tb{
- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX2__)
@@ -1032,8 +1000,8 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
if (Btype != GGML_TYPE_F32)
return false;
- tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, float> tb{
- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, TC> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
@@ -1041,8 +1009,8 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
if (Btype != GGML_TYPE_F32)
return false;
- tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, float> tb{
- k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, TC> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
@@ -1055,15 +1023,15 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
if (k % 16)
return false;
if (Btype == GGML_TYPE_F32 && n < 2) {
- tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, TC> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
}
if (Btype != GGML_TYPE_F16)
return false;
- tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, TC> tb{
+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
@@ -1071,17 +1039,15 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
if (k % 8)
return false;
if (Btype == GGML_TYPE_F32 && n < 2) {
- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith,
- nth};
+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, TC> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
}
if (Btype != GGML_TYPE_F16)
return false;
- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith,
- nth};
+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, TC> tb{
+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
} else {
@@ -1091,14 +1057,25 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
if (n < 2)
// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?
return false;
- if (k % 8)
- return false;
- if (Btype != GGML_TYPE_F16)
- return false;
- tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
- return true;
+ if (precision == GGML_PREC_F32) {
+ if (k % 4)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+ } else {
+ if (k % 8)
+ return false;
+ if (Btype != GGML_TYPE_F16)
+ return false;
+ tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, TC> tb{
+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+ }
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
if (n < 2)
// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?
@@ -1107,8 +1084,8 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
if (Btype != GGML_TYPE_F32)
return false;
- tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
@@ -1120,13 +1097,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
if (Btype != GGML_TYPE_Q8_0)
return false;
#if defined(__AVX2__) || defined(__AVX512F__)
- tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, float> tb{
- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, TC> tb{
+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_FEATURE_DOTPROD)
- tinyBLAS_Q0_ARM<0, block_q8_0, block_q8_0, float> tb{
- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS_Q0_ARM<0, block_q8_0, block_q8_0, TC> tb{
+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
@@ -1138,13 +1115,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
if (Btype != GGML_TYPE_Q8_0)
return false;
#if defined(__AVX2__) || defined(__AVX512F__)
- tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, float> tb{
- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, TC> tb{
+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_FEATURE_DOTPROD)
- tinyBLAS_Q0_ARM<0, block_q4_0, block_q8_0, float> tb{
- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tinyBLAS_Q0_ARM<0, block_q4_0, block_q8_0, TC> tb{
+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
@@ -1169,7 +1146,65 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
(void)nth;
(void)Atype;
(void)Btype;
- (void)Ctype;
+ (void)precision;
+}
+
+} // namespace
+
+/**
+ * Performs optimized matrix multiplication on CPU.
+ *
+ * This subroutine may compute C = Aᵀ * B with column major ordering.
+ * Despite its name, this isn't a generalized implementation. Work is
+ * only performed when a handwritten kernel is written and available.
+ * Otherwise the caller should fall back to a general matmul routine.
+ *
+ * For example, for single-threaded single-precision GEMM you can say
+ *
+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1,
+ * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32,
+ * GGML_PREC_DEFAULT);
+ *
+ * @param m is rows in `A` and `C`
+ * @param n is cols in `B` and `C`
+ * @param k is cols in `A` and rows in `B`
+ * @param A is first input matrix (always transposed)
+ * @param lda is row stride of `A`
+ * @param B is second input matrix (never transposed)
+ * @param ldb is row stride of `B`
+ * @param C is input/output array of output matrices
+ * @param ldc is row stride of `C`
+ * @param ith is thread id (must be less than `nth`)
+ * @param nth is number of threads (must be greater than zero)
+ * @param Atype is GGML data type of `A`
+ * @param Btype is GGML data type of `B`
+ * @param Ctype is GGML data type of `C`
+ * @param precision may be used to control the internal compute type
+ * @return true if this function was able to service the matmul request
+ */
+bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,
+ void *C, long ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype,
+ int precision) {
+
+ assert(m >= 0);
+ assert(n >= 0);
+ assert(k >= 0);
+ assert(lda >= k);
+ assert(ldb >= k);
+ assert(ldc >= m);
+ assert(nth > 0);
+ assert(ith < nth);
+
+ switch (Ctype) {
+ case GGML_TYPE_F32:
+ return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (float *)C, ldc, ith, nth, task, Atype,
+ Btype, precision);
+ case GGML_TYPE_F16:
+ return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_fp16_t *)C, ldc, ith, nth, task,
+ Atype, Btype, precision);
+ default:
+ return false;
+ }
}
//
@@ -1285,15 +1320,12 @@ class MixMul {
assert(params->ith < params->nth);
assert(plan->type == GGML_TYPE_I32);
- // supported types
- if (result->type != GGML_TYPE_F32)
- return false;
-
// check nb01 is convertible to lda
if (weights->nb[1] % ggml_type_size(weights->type))
return false;
// no support for column strides
+
if (result->nb[0] != ggml_type_size(result->type))
return false;
if (thought->nb[0] != ggml_type_size(thought->type))
@@ -1301,24 +1333,37 @@ class MixMul {
if (weights->nb[0] != ggml_type_size(weights->type))
return false;
+ // supported output types
+ switch (result->type) {
+ case GGML_TYPE_F32:
+ return mixmuler<float>();
+ case GGML_TYPE_F16:
+ return mixmuler<ggml_fp16_t>();
+ default:
+ return false;
+ }
+ }
+
+ private:
+ template <typename TC>
+ bool mixmuler() {
switch (weights->type) {
case GGML_TYPE_F32:
if (thought->type != GGML_TYPE_F32)
return false;
#if defined(__AVX512F__)
- return mixmat<16, 1, tinyBLAS<NCB | NCC, 16, __m512, __m512, float, float, float>,
- float, float, float>();
+ return mixmat<16, 1, tinyBLAS<NCB | NCC, 16, __m512, __m512, float, float, TC>, float,
+ float, TC>();
#elif defined(__AVX__) || defined(__AVX2__)
- return mixmat<8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, float, float, float>, float,
- float, float>();
+ return mixmat<8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, float, float, TC>, float,
+ float, TC>();
#elif defined(__SSE__)
- return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, __m128, __m128, float, float, float>, float,
- float, float>();
+ return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, __m128, __m128, float, float, TC>, float,
+ float, TC>();
#elif defined(__ARM_NEON)
- return mixmat<4, 1,
- tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, float, float, float>,
- float, float, float>();
+ return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, float, float, TC>,
+ float, float, TC>();
#else
return false;
#endif
@@ -1327,22 +1372,22 @@ class MixMul {
if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_BF16)
return false;
#if defined(__AVX512BF16__)
- return mixmat<
- 32, 1, tinyBLAS<NCB | NCC, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float>,
- ggml_bf16_t, ggml_bf16_t, float>();
+ return mixmat<32, 1,
+ tinyBLAS<NCB | NCC, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC>,
+ ggml_bf16_t, ggml_bf16_t, TC>();
#elif defined(__AVX512F__)
return mixmat<16, 1,
- tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, float>,
- ggml_bf16_t, ggml_bf16_t, float>();
+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, TC>,
+ ggml_bf16_t, ggml_bf16_t, TC>();
#elif defined(__AVX2__)
return mixmat<8, 1,
- tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, float>,
- ggml_bf16_t, ggml_bf16_t, float>();
+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, TC>,
+ ggml_bf16_t, ggml_bf16_t, TC>();
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
return mixmat<
4, 1,
- tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float>,
- ggml_bf16_t, ggml_bf16_t, float>();
+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, TC>,
+ ggml_bf16_t, ggml_bf16_t, TC>();
#else
return false;
#endif
@@ -1352,26 +1397,33 @@ class MixMul {
return false;
#if defined(__AVX512F__)
return mixmat<16, 1,
- tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float>,
- ggml_fp16_t, ggml_fp16_t, float>();
+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, TC>,
+ ggml_fp16_t, ggml_fp16_t, TC>();
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
if (X86_HAVE(F16C)) {
- return mixmat<
- 8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,
- ggml_fp16_t, ggml_fp16_t, float>();
+ return mixmat<8, 1,
+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, TC>,
+ ggml_fp16_t, ggml_fp16_t, TC>();
} else {
return false;
}
#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
- return mixmat<
- 8, 1,
- tinyBLAS<NCB | NCC, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float>,
- ggml_fp16_t, ggml_fp16_t, float>();
+ if (result->op_params[0] == GGML_PREC_F32) {
+ return mixmat<
+ 4, 1,
+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, TC>,
+ ggml_fp16_t, ggml_fp16_t, TC>();
+ } else {
+ return mixmat<
+ 8, 1,
+ tinyBLAS<NCB | NCC, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, TC>,
+ ggml_fp16_t, ggml_fp16_t, TC>();
+ }
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
return mixmat<
4, 1,
- tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, float>,
- ggml_fp16_t, ggml_fp16_t, float>();
+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, TC>,
+ ggml_fp16_t, ggml_fp16_t, TC>();
#else
return false;
#endif
@@ -1380,11 +1432,11 @@ class MixMul {
if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)
return false;
#if defined(__AVX2__) || defined(__AVX512F__)
- return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q4_0, block_q8_0, float>,
- block_q4_0, block_q8_0, float>();
+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q4_0, block_q8_0, TC>,
+ block_q4_0, block_q8_0, TC>();
#elif defined(__ARM_FEATURE_DOTPROD)
- return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0, block_q8_0, float>,
- block_q4_0, block_q8_0, float>();
+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0, block_q8_0, TC>,
+ block_q4_0, block_q8_0, TC>();
#else
return false;
#endif
@@ -1393,11 +1445,11 @@ class MixMul {
if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)
return false;
#if defined(__AVX2__) || defined(__AVX512F__)
- return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q8_0, block_q8_0, float>,
- block_q8_0, block_q8_0, float>();
+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q8_0, block_q8_0, TC>,
+ block_q8_0, block_q8_0, TC>();
#elif defined(__ARM_FEATURE_DOTPROD)
- return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0, block_q8_0, float>,
- block_q8_0, block_q8_0, float>();
+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0, block_q8_0, TC>,
+ block_q8_0, block_q8_0, TC>();
#else
return false;
#endif
@@ -1407,7 +1459,6 @@ class MixMul {
}
}
- private:
template <int KN, int BS, typename BLAS, typename TA, typename TB, typename TC>
bool mixmat() {
if (cols % KN)