Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Synchronize with llama.cpp upstream
Justine Tunney820d42d7db7f
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex e5ee79f5..12763a69 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -8,30 +8,17 @@ LICENSEORIGIN- Sources were initially imported from:-- https://github.com/ggerganov/llama.cpp- e9c1cecb9d7d743d30b4a29ecd56a411437def0a- Tue Nov 7 09:04:51 2023 +0100-- Since then, we've cherry-picked a number of small changes which fix- bugs and add features. The changes we haven't cherry-picked yet are- Georgi and Slaren's active development work, improving tensor graph- allocation, scheduling, backends, etc. So far, we've curated up to:-- https://github.com/ggerganov/llama.cpp- fe680e3d1080a765e5d3150ffd7bab189742898d- Thu Dec 7 22:26:54 2023 +0200-- See the llamafile git history re: cherry-picks for further details.+ https://github.com/ggerganov/llama.cpp/pull/4406/+ e1241d9b461816b679eaf6951631287687a18f66+ 2023-12-13LOCAL MODIFICATIONS- Refactor ggml.c, llama.cpp, and llava to use llamafile_open() APIs+ - Make cuBLAS dependency optional by introducing tinyBLAS library- Add support to main() programs for Cosmo /zip/.args files- - Use condition variables for server rather than busy loop- Introduce pledge() SECCOMP sandboxing to improve security- - Introduce ggml_interrupt() to gracefully handle signals+ - Use condition variables for server rather than busy loop- Remove log callback pointer API from Metal GPU module- Avoid bind() conflicts on port 8080 w/ server- Allow --grammar to be used on --image promptsdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 1d4f4862..45f4a606 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -18,6 +18,7 @@#include <regex>#include <sstream>#include <string>+#include <unordered_map>#include <unordered_set>#include <vector>#include <cinttypes>@@ -288,8 +289,18 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {break;}params.yarn_beta_slow = std::stof(argv[i]);- } else if (arg == "--memory-f32") {- params.memory_f16 = false;+ } else if (arg == "--samplers") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ sparams.samplers_sequence = parse_samplers_input(argv[i]);+ } else if (arg == "--sampling-seq") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ sparams.samplers_sequence = argv[i];} else if (arg == "--top-p") {if (++i >= argc) {invalid_param = true;@@ -502,10 +513,20 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {params.interactive_first = true;} else if (arg == "-ins" || arg == "--instruct") {params.instruct = true;+ } else if (arg == "-cml" || arg == "--chatml") {+ params.chatml = true;} else if (arg == "--infill") {params.infill = true;} else if (arg == "--unsecure") {params.unsecure = true;+ } else if (arg == "-dkvc" || arg == "--dump-kv-cache") {+ params.dump_kv_cache = true;+ } else if (arg == "-nkvo" || arg == "--no-kv-offload") {+ params.no_kv_offload = true;+ } else if (arg == "-ctk" || arg == "--cache-type-k") {+ params.cache_type_k = argv[++i];+ } else if (arg == "-ctv" || arg == "--cache-type-v") {+ params.cache_type_v = argv[++i];} else if (arg == "--multiline-input") {params.multiline_input = true;} else if (arg == "--simple-io") {@@ -685,6 +706,47 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {std::istreambuf_iterator<char>(),std::back_inserter(sparams.grammar));+ } else if (arg == "--override-kv") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ char * sep = strchr(argv[i], '=');+ if (sep == nullptr || sep - argv[i] >= 128) {+ fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);+ invalid_param = true;+ break;+ }+ struct llama_model_kv_override kvo;+ std::strncpy(kvo.key, argv[i], sep - argv[i]);+ kvo.key[sep - argv[i]] = 0;+ sep++;+ if (strncmp(sep, "int:", 4) == 0) {+ sep += 4;+ kvo.tag = LLAMA_KV_OVERRIDE_INT;+ kvo.int_value = std::atol(sep);+ } else if (strncmp(sep, "float:", 6) == 0) {+ sep += 6;+ kvo.tag = LLAMA_KV_OVERRIDE_FLOAT;+ kvo.float_value = std::atof(sep);+ } else if (strncmp(sep, "bool:", 5) == 0) {+ sep += 5;+ kvo.tag = LLAMA_KV_OVERRIDE_BOOL;+ if (std::strcmp(sep, "true") == 0) {+ kvo.bool_value = true;+ } else if (std::strcmp(sep, "false") == 0) {+ kvo.bool_value = false;+ } else {+ fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);+ invalid_param = true;+ break;+ }+ } else {+ fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);+ invalid_param = true;+ break;+ }+ params.kv_overrides.push_back(kvo);#ifndef LOG_DISABLE_LOGS// Parse args for logging parameters} else if ( log_param_single_parse( argv[i] ) ) {@@ -728,6 +790,11 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {}}+ if (!params.kv_overrides.empty()) {+ params.kv_overrides.emplace_back(llama_model_kv_override());+ params.kv_overrides.back().key[0] = 0;+ }+if (passed_gpu_flags) {// user is tuning their gpuif (!ggml_metal_supported() && !ggml_cuda_supported()) {@@ -759,6 +826,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -i, --interactive run in interactive mode\n");printf(" --interactive-first run in interactive mode and wait for input right away\n");printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n");+ printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n");printf(" --multiline-input allows you to write or paste multiple lines without ending each in '\\'\n");printf(" -r PROMPT, --reverse-prompt PROMPT\n");printf(" halt generation at PROMPT, return control in interactive mode\n");@@ -784,6 +852,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -n N, --n-predict N number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)\n", params.n_predict);printf(" -c N, --ctx-size N size of the prompt context (default: %d, 0 = loaded from model)\n", params.n_ctx);printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);+ printf(" --samplers samplers that will be used for generation in the order, separated by \';\', for example: \"top_k;tfs;typical;top_p;min_p;temp\"\n");+ printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sparams.samplers_sequence.c_str());printf(" --top-k N top-k sampling (default: %d, 0 = disabled)\n", sparams.top_k);printf(" --top-p N top-p sampling (default: %.1f, 1.0 = disabled)\n", (double)sparams.top_p);printf(" --min-p N min-p sampling (default: %.1f, 0.0 = disabled)\n", (double)sparams.min_p);@@ -821,8 +891,6 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast);printf(" --ignore-eos ignore end of stream token and continue generating (implies --logit-bias 2-inf)\n");printf(" --no-penalize-nl do not penalize newline token\n");- printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n");- printf(" not recommended: doubles context memory required and no measurable increase in quality\n");printf(" --temp N temperature (default: %.1f)\n", (double)sparams.temp);printf(" --logits-all return logits for all tokens in the batch (default: disabled)\n");printf(" --hellaswag compute HellaSwag score over random tasks from datafile supplied with -f\n");@@ -857,6 +925,14 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" use " GGML_CUBLAS_NAME " instead of custom mul_mat_q " GGML_CUDA_NAME " kernels.\n");printf(" Not recommended since this is both slower and uses more VRAM.\n");printf(" --verbose-prompt print prompt before generation\n");+ printf(" -dkvc, --dump-kv-cache\n");+ printf(" verbose print of the KV cache\n");+ printf(" -nkvo, --no-kv-offload\n");+ printf(" disable KV offload\n");+ printf(" -ctk TYPE, --cache-type-k TYPE\n");+ printf(" KV cache data type for K (default: %s)\n", params.cache_type_k.c_str());+ printf(" -ctv TYPE, --cache-type-v TYPE\n");+ printf(" KV cache data type for V (default: %s)\n", params.cache_type_v.c_str());printf(" --simple-io use basic IO for better compatibility in subprocesses and limited consoles\n");printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");printf(" --lora-scaled FNAME S apply LoRA adapter with user defined scaling S (implies --no-mmap)\n");@@ -868,6 +944,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -ld LOGDIR, --logdir LOGDIR\n");printf(" path under which to save YAML logs (no logging if unset)\n");printf(" --unsecure disables pledge() sandboxing on Linux and OpenBSD\n");+ printf(" --override-kv KEY=TYPE:VALUE\n");+ printf(" advanced option to override model metadata by key. may be specified multiple times.\n");+ printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");printf("\n");#ifndef LOG_DISABLE_LOGSlog_print_usage();@@ -904,6 +983,48 @@ std::string gpt_random_prompt(std::mt19937 & rng) {GGML_UNREACHABLE();}+//+// String parsing+//++std::string parse_samplers_input(std::string input) {+ std::string output = "";+ // since samplers names are written multiple ways+ // make it ready for both system names and input names+ std::unordered_map<std::string, char> samplers_symbols {+ {"top_k", 'k'},+ {"top-k", 'k'},+ {"top_p", 'p'},+ {"top-p", 'p'},+ {"nucleus", 'p'},+ {"typical_p", 'y'},+ {"typical-p", 'y'},+ {"typical", 'y'},+ {"min_p", 'm'},+ {"min-p", 'm'},+ {"tfs_z", 'f'},+ {"tfs-z", 'f'},+ {"tfs", 'f'},+ {"temp", 't'},+ {"temperature",'t'}+ };+ // expected format example: "temp;top_k;tfs_z;typical_p;top_p;min_p"+ size_t separator = input.find(';');+ while (separator != input.npos) {+ std::string name = input.substr(0,separator);+ input = input.substr(separator+1);+ separator = input.find(';');++ if (samplers_symbols.find(name) != samplers_symbols.end()) {+ output += samplers_symbols[name];+ }+ }+ if (samplers_symbols.find(input) != samplers_symbols.end()) {+ output += samplers_symbols[input];+ }+ return output;+}+//// Model utils//@@ -918,10 +1039,39 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &mparams.tensor_split = params.tensor_split;mparams.use_mmap = params.use_mmap;mparams.use_mlock = params.use_mlock;+ if (params.kv_overrides.empty()) {+ mparams.kv_overrides = NULL;+ } else {+ GGML_ASSERT(params.kv_overrides.back().key[0] == 0 && "KV overrides not terminated with empty key");+ mparams.kv_overrides = params.kv_overrides.data();+ }return mparams;}+static ggml_type kv_cache_type_from_str(const std::string & s) {+ if (s == "f16") {+ return GGML_TYPE_F16;+ }+ if (s == "q8_0") {+ return GGML_TYPE_Q8_0;+ }+ if (s == "q4_0") {+ return GGML_TYPE_Q4_0;+ }+ if (s == "q4_1") {+ return GGML_TYPE_Q4_1;+ }+ if (s == "q5_0") {+ return GGML_TYPE_Q5_0;+ }+ if (s == "q5_1") {+ return GGML_TYPE_Q5_1;+ }++ ThrowRuntimeError("Invalid cache type: " + s);+}+struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params) {auto cparams = llama_context_default_params();@@ -931,7 +1081,6 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_paramcparams.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch;cparams.mul_mat_q = params.mul_mat_q;cparams.seed = params.seed;- cparams.f16_kv = params.memory_f16;cparams.logits_all = params.logits_all;cparams.embedding = params.embedding;cparams.rope_scaling_type = params.rope_scaling_type;@@ -942,6 +1091,10 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_paramcparams.yarn_beta_fast = params.yarn_beta_fast;cparams.yarn_beta_slow = params.yarn_beta_slow;cparams.yarn_orig_ctx = params.yarn_orig_ctx;+ cparams.offload_kqv = !params.no_kv_offload;++ cparams.type_k = kv_cache_type_from_str(params.cache_type_k);+ cparams.type_v = kv_cache_type_from_str(params.cache_type_v);return cparams;}@@ -957,7 +1110,7 @@ void llama_batch_add(const std::vector<llama_seq_id> & seq_ids,bool logits) {batch.token [batch.n_tokens] = id;- batch.pos [batch.n_tokens] = pos,+ batch.pos [batch.n_tokens] = pos;batch.n_seq_id[batch.n_tokens] = seq_ids.size();for (size_t i = 0; i < seq_ids.size(); ++i) {batch.seq_id[batch.n_tokens][i] = seq_ids[i];@@ -1098,80 +1251,19 @@ std::string llama_detokenize_bpe(llama_context * ctx, const std::vector<llama_toreturn result;}+bool llama_should_add_bos_token(const llama_model * model) {+ const int add_bos = llama_add_bos_token(model);++ return add_bos != -1 ? bool(add_bos) : (llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM);+}+//// YAML utils//// returns true if successful, false otherwisebool create_directory_with_parents(const std::string & path) {-#ifdef _WIN32- std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;- std::wstring wpath = converter.from_bytes(path);-- // if the path already exists, check whether it's a directory- const DWORD attributes = GetFileAttributesW(wpath.c_str());- if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {- return true;- }-- size_t pos_slash = 0;-- // process path from front to back, procedurally creating directories- while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {- const std::wstring subpath = wpath.substr(0, pos_slash);- const wchar_t * test = subpath.c_str();-- const bool success = CreateDirectoryW(test, NULL);- if (!success) {- const DWORD error = GetLastError();-- // if the path already exists, ensure that it's a directory- if (error == ERROR_ALREADY_EXISTS) {- const DWORD attributes = GetFileAttributesW(subpath.c_str());- if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {- return false;- }- } else {- return false;- }- }-- pos_slash += 1;- }-- return true;-#else- // if the path already exists, check whether it's a directory- struct stat info;- if (stat(path.c_str(), &info) == 0) {- return S_ISDIR(info.st_mode);- }-- size_t pos_slash = 1; // skip leading slashes for directory creation-- // process path from front to back, procedurally creating directories- while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {- const std::string subpath = path.substr(0, pos_slash);- struct stat info;-- // if the path already exists, ensure that it's a directory- if (stat(subpath.c_str(), &info) == 0) {- if (!S_ISDIR(info.st_mode)) {- return false;- }- } else {- // create parent directories- const int ret = mkdir(subpath.c_str(), 0755);- if (ret != 0) {- return false;- }- }-- pos_slash += 1;- }-- return true;-#endif // _WIN32+ return !makedirs(path.c_str(), 0755);}void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector<float> & data) {@@ -1214,6 +1306,7 @@ void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const chaif (!data_str.empty() && (std::isspace(data_str[0]) || std::isspace(data_str.back()))) {data_str = std::regex_replace(data_str, std::regex("\n"), "\\n");data_str = std::regex_replace(data_str, std::regex("\""), "\\\"");+ data_str = std::regex_replace(data_str, std::regex(R"(\\[^n"])"), R"(\$&)");data_str = "\"" + data_str + "\"";fprintf(stream, "%s: %s\n", prop_name, data_str.c_str());return;@@ -1242,7 +1335,7 @@ std::string get_sortable_timestamp() {const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(current_time.time_since_epoch() % 1000000000).count();char timestamp_ns[21];- snprintf(timestamp_ns, 21, "%09" PRId64, ns);+ snprintf(timestamp_ns, 11, "%09" PRId64, ns);return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns);}@@ -1347,7 +1440,6 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l}fprintf(stream, "lora_base: %s\n", params.lora_base.c_str());fprintf(stream, "main_gpu: %d # default: 0\n", params.main_gpu);- fprintf(stream, "memory_f32: %s # default: false\n", !params.memory_f16 ? "true" : "false");fprintf(stream, "mirostat: %d # default: 0 (disabled)\n", sparams.mirostat);fprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau);fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta);@@ -1402,3 +1494,77 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "typical_p: %f # default: 1.0\n", sparams.typical_p);fprintf(stream, "verbose_prompt: %s # default: false\n", params.verbose_prompt ? "true" : "false");}++//+// KV cache utils+//++void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size) {+ static const char slot_chars[] = ".123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+";++ printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d",+ view.n_cells, view.n_max_seq, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);++ llama_kv_cache_view_cell * c_curr = view.cells;+ llama_seq_id * cs_curr = view.cells_sequences;++ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) {+ if (i % row_size == 0) {+ printf("\n%5d: ", i);+ }+ int seq_count = 0;+ for (int j = 0; j < view.n_max_seq; j++) {+ if (cs_curr[j] >= 0) { seq_count++; }+ }+ putchar(slot_chars[std::min(sizeof(slot_chars) - 2, size_t(seq_count))]);+ }++ printf("\n=== Done dumping\n");+}++void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) {+ static const char slot_chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";++ printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d\n",+ view.n_cells, view.n_max_seq, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);++ std::unordered_map<llama_seq_id, size_t> seqs;+ llama_kv_cache_view_cell * c_curr = view.cells;+ llama_seq_id * cs_curr = view.cells_sequences;++ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) {+ for (int j = 0; j < view.n_max_seq; j++) {+ if (cs_curr[j] < 0) { continue; }+ if (seqs.find(cs_curr[j]) == seqs.end()) {+ if (seqs.size() + 1 >= sizeof(slot_chars)) { break; }+ seqs[cs_curr[j]] = seqs.size();+ }+ }+ if (seqs.size() + 1 >= sizeof(slot_chars)) { break; }+ }++ printf("=== Sequence legend: ");+ for (const auto & it : seqs) {+ printf("%zu=%d, ", it.second, it.first);+ }+ printf("'+'=other sequence ids");++ c_curr = view.cells;+ cs_curr = view.cells_sequences;+ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) {+ if (i % row_size == 0) {+ printf("\n%5d: ", i);+ }+ for (int j = 0; j < view.n_max_seq; j++) {+ if (cs_curr[j] >= 0) {+ const auto & it = seqs.find(cs_curr[j]);+ putchar(it != seqs.end() ? int(slot_chars[it->second]) : '+');+ } else {+ putchar('.');+ }+ }+ putchar(' ');+ }++ printf("\n=== Done dumping\n");+}diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 874fcbdc..cc0ff163 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -1,8 +1,12 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+// Various helper functions and utilities#pragma once#include "llama.h"+#include "sampling.h"#include "llamafile/version.h"@@ -85,6 +89,8 @@ struct gpt_params {std::vector<std::string> antiprompt; // string upon seeing which more user input is promptedstd::string logdir = ""; // directory in which to save YAML log files+ std::vector<llama_model_kv_override> kv_overrides;+// TODO: avoid tuple, use structstd::vector<std::tuple<std::string, float>> lora_adapter; // lora adapter path with user defined scalestd::string lora_base = ""; // base model path for the lora adapter@@ -97,10 +103,10 @@ struct gpt_params {size_t hellaswag_tasks = 400; // number of tasks to use when computing the HellaSwag scorebool mul_mat_q = true; // if true, use mul_mat_q kernels instead of cuBLAS- bool memory_f16 = true; // use f16 instead of f32 for memory kvbool random_prompt = false; // do not randomize prompt if none providedbool use_color = false; // use color to distinguish generations and inputsbool interactive = false; // interactive mode+ bool chatml = false; // chatml mode (used for models trained on chatml syntax)bool prompt_cache_all = false; // save user input and generations to prompt cachebool prompt_cache_ro = false; // open the prompt cache read-only and do not update it@@ -121,12 +127,16 @@ struct gpt_params {bool verbose_prompt = false; // print prompt tokens before generationbool silent_prompt = false; // don't print prompt to stdoutbool infill = false; // use infill mode+ bool dump_kv_cache = false; // dump the KV cache contents for debugging purposes+ bool no_kv_offload = false; // disable KV offloading- bool unsecure = false; // disable pledge() sandboxing+ bool unsecure = false; // disable pledge() sandboxing+ std::string cache_type_k = "f16"; // KV cache data type for the K+ std::string cache_type_v = "f16"; // KV cache data type for the V// multimodal models (see examples/llava)std::string mmproj = ""; // path to multimodal projector- std::string image = ""; // path to an image file+ std::string image = ""; // path to an image file};bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params);@@ -141,6 +151,12 @@ std::string gpt_random_prompt(std::mt19937 & rng);void process_escapes(std::string& input);+//+// String parsing+//++std::string parse_samplers_input(std::string input);+//// Model utils//@@ -202,6 +218,10 @@ std::string llama_detokenize_bpe(llama_context * ctx,const std::vector<llama_token> & tokens);+// Uses the value from the model metadata if possible, otherwise+// defaults to true when model type is SPM, otherwise false.+bool llama_should_add_bos_token(const llama_model * model);+//// YAML utils//@@ -215,3 +235,13 @@ std::string get_sortable_timestamp();void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const llama_context * lctx,const std::string & timestamp, const std::vector<int> & prompt_tokens, const char * model_desc);++//+// KV cache utils+//++// Dump the KV cache view with the number of sequences per cell.+void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size = 80);++// Dump the KV cache view showing individual sequences in each cell (long output).+void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40);diff --git a/llama.cpp/console.cpp b/llama.cpp/console.cppindex 4e793930..b6aa53c2 100644--- a/llama.cpp/console.cpp+++ b/llama.cpp/console.cpp@@ -3,6 +3,7 @@#include "console.h"#include <vector>#include <iostream>+#include <climits>#include <sys/ioctl.h>#include <unistd.h>diff --git a/llama.cpp/ggml-alloc.c b/llama.cpp/ggml-alloc.cindex 34eba3f8..1c3d95c2 100644--- a/llama.cpp/ggml-alloc.c+++ b/llama.cpp/ggml-alloc.c@@ -1,51 +1,23 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi#include "ggml-alloc.h"-#include "ggml-backend.h"+#include "ggml-backend-impl.h"#include "ggml.h"+#include "ggml-impl.h"#include <assert.h>+#include <limits.h>#include <stdarg.h>#include <stdio.h>#include <stdlib.h>#include <string.h>--#define UNUSED(x) (void)(x)#define MAX(a, b) ((a) > (b) ? (a) : (b))-#define GGML_MAX_CONCUR (2*GGML_MAX_NODES)+#define MAX_FREE_BLOCKS 256//#define GGML_ALLOCATOR_DEBUG-//#define AT_PRINTF printf-#define AT_PRINTF(...) ((void)0)--struct hash_node {- struct ggml_tensor * t;- int n_children;- int n_views;-};--static size_t hash(void * p) {- return (size_t)p % GGML_GRAPH_HASHTABLE_SIZE;-}--static struct hash_node * hash_get(struct hash_node hash_table[], struct ggml_tensor * t) {- size_t h = hash(t);-- // linear probing- size_t i = h;- while (hash_table[i].t != NULL) {- if (hash_table[i].t == t) {- return &hash_table[i];- }- i = (i + 1) % GGML_GRAPH_HASHTABLE_SIZE;- if (i == h) {- // hash table is full- GGML_ASSERT(false);- }- }-- hash_table[i].t = t;- return &hash_table[i];-}+//#define AT_PRINTF(...) fprintf(stderr, __VA_ARGS__)+#define AT_PRINTF(...)// TODO: GGML_PAD ?static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {@@ -59,20 +31,18 @@ struct free_block {size_t size;};-#define MAX_FREE_BLOCKS 256--struct ggml_allocr {+struct ggml_tallocr {struct ggml_backend_buffer * buffer;bool buffer_owned;- void * data;+ void * base;size_t alignment;+int n_free_blocks;struct free_block free_blocks[MAX_FREE_BLOCKS];- struct hash_node hash_table[GGML_GRAPH_HASHTABLE_SIZE];+size_t max_size;+bool measure;- int parse_seq[GGML_MAX_CONCUR];- int parse_seq_len;#ifdef GGML_ALLOCATOR_DEBUGstruct ggml_tensor * allocated_tensors[1024];@@ -80,7 +50,7 @@ struct ggml_allocr {};#ifdef GGML_ALLOCATOR_DEBUG-static void add_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {+static void add_allocated_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {for (int i = 0; i < 1024; i++) {if (alloc->allocated_tensors[i] == NULL) {alloc->allocated_tensors[i] = tensor;@@ -89,7 +59,7 @@ static void add_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tensor}GGML_ASSERT(!"out of allocated_tensors");}-static void remove_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {+static void remove_allocated_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {for (int i = 0; i < 1024; i++) {if (alloc->allocated_tensors[i] == tensor ||(alloc->allocated_tensors[i] != NULL && alloc->allocated_tensors[i]->data == tensor->data)) {@@ -103,7 +73,7 @@ static void remove_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tens#endif// check if a tensor is allocated by this buffer-static bool ggml_allocr_is_own(struct ggml_allocr * alloc, const struct ggml_tensor * tensor) {+static bool ggml_tallocr_is_own(ggml_tallocr_t alloc, const struct ggml_tensor * tensor) {return tensor->buffer == alloc->buffer;}@@ -111,7 +81,7 @@ static bool ggml_is_view(struct ggml_tensor * t) {return t->view_src != NULL;}-void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {+void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {GGML_ASSERT(!ggml_is_view(tensor)); // views generally get data pointer from one of their sourcesGGML_ASSERT(tensor->data == NULL); // avoid allocating tensor which already has memory allocated@@ -162,13 +132,14 @@ void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor)}tensor->data = addr;- AT_PRINTF("%s: allocated data at %p\n", __func__, tensor->data);tensor->buffer = alloc->buffer;- ggml_backend_buffer_init_tensor(alloc->buffer, tensor);+ if (!alloc->measure) {+ ggml_backend_buffer_init_tensor(alloc->buffer, tensor);+ }#ifdef GGML_ALLOCATOR_DEBUGadd_allocated_tensor(alloc, tensor);- size_t cur_max = (char*)addr - (char*)alloc->data + size;+ size_t cur_max = (char*)addr - (char*)alloc->base + size;if (cur_max > alloc->max_size) {printf("max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0);for (int i = 0; i < 1024; i++) {@@ -180,16 +151,16 @@ void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor)}#endif- alloc->max_size = MAX(alloc->max_size, (char*)addr - (char*)alloc->data + size);+ alloc->max_size = MAX(alloc->max_size, (char*)addr - (char*)alloc->base + size);}// this is a very naive implementation, but for our case the number of free blocks should be very small-static void ggml_allocr_free_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {- if (ggml_allocr_is_own(alloc, tensor) == false) {+static void ggml_tallocr_free_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {+ if (ggml_tallocr_is_own(alloc, tensor) == false) {// the tensor was not allocated in this buffer// this can happen because the graph allocator will try to free weights and other tensors from different buffers// the easiest way to deal with this is just to ignore it- AT_PRINTF("ignoring %s (their buffer: %p, our buffer: %p)\n", tensor->name, (void *)tensor->buffer, (void *)alloc->buffer);+ // AT_PRINTF("ignoring %s (their buffer: %p, our buffer: %p)\n", tensor->name, (void *)tensor->buffer, (void *)alloc->buffer);return;}@@ -199,8 +170,6 @@ static void ggml_allocr_free_tensor(struct ggml_allocr * alloc, struct ggml_tenssize = aligned_offset(NULL, size, alloc->alignment);AT_PRINTF("%s: freeing %s at %p (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, ptr, size, alloc->n_free_blocks);- ggml_backend_buffer_free_tensor(alloc->buffer, tensor);-#ifdef GGML_ALLOCATOR_DEBUGremove_allocated_tensor(alloc, tensor);#endif@@ -253,91 +222,180 @@ static void ggml_allocr_free_tensor(struct ggml_allocr * alloc, struct ggml_tensalloc->n_free_blocks++;}-void ggml_allocr_set_parse_seq(struct ggml_allocr * alloc, const int * list, int n) {- for (int i = 0; i < n; i++) {- alloc->parse_seq[i] = list[i];- }- alloc->parse_seq_len = n;-}--void ggml_allocr_reset(struct ggml_allocr * alloc) {+void ggml_tallocr_reset(ggml_tallocr_t alloc) {alloc->n_free_blocks = 1;- size_t align_offset = aligned_offset(alloc->data, 0, alloc->alignment);- alloc->free_blocks[0].addr = (char *)alloc->data + align_offset;- alloc->free_blocks[0].size = ggml_backend_buffer_get_size(alloc->buffer) - align_offset;+ size_t align_offset = aligned_offset(alloc->base, 0, alloc->alignment);+ alloc->free_blocks[0].addr = (char *)alloc->base + align_offset;++ if (alloc->measure) {+ alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows+ } else {+ alloc->free_blocks[0].size = ggml_backend_buffer_get_size(alloc->buffer) - align_offset;+ }}-struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment) {- struct ggml_backend_buffer * buffer = ggml_backend_cpu_buffer_from_ptr(NULL, data, size);+ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment) {+ struct ggml_backend_buffer * buffer = ggml_backend_cpu_buffer_from_ptr(data, size);- struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr));+ ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr));- *alloc = (struct ggml_allocr){+ *alloc = (struct ggml_tallocr) {/*.buffer = */ buffer,/*.buffer_owned = */ true,/*.base = */ ggml_backend_buffer_get_base(buffer),/*.alignment = */ alignment,/*.n_free_blocks = */ 0,/*.free_blocks = */ {{0}},- /*.hash_table = */ {{0}},/*.max_size = */ 0,/*.measure = */ false,- /*.parse_seq = */ {0},- /*.parse_seq_len = */ 0,#ifdef GGML_ALLOCATOR_DEBUG/*.allocated_tensors = */ {0},#endif};- ggml_allocr_reset(alloc);+ ggml_tallocr_reset(alloc);return alloc;}-struct ggml_allocr * ggml_allocr_new_measure(size_t alignment) {- struct ggml_allocr * alloc = ggml_allocr_new((void *)0x1000, (size_t)-0x1001, alignment);+ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment) {+ ggml_tallocr_t alloc = ggml_tallocr_new((void *)0x1000, SIZE_MAX/2, alignment);alloc->measure = true;return alloc;}-struct ggml_allocr * ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer) {- struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr));+ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend) {+ // create a backend buffer to get the correct tensor allocation sizes+ ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, 1);++ // TODO: move alloc initialization to a common ggml_tallocr_new_impl function+ ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer);+ alloc->buffer_owned = true;+ alloc->measure = true;+ ggml_tallocr_reset(alloc);+ return alloc;+}++ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size) {+ ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, size);+ ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer);+ alloc->buffer_owned = true;+ return alloc;+}- *alloc = (struct ggml_allocr){+ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer) {+ ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr));++ *alloc = (struct ggml_tallocr) {/*.buffer = */ buffer,/*.buffer_owned = */ false,/*.base = */ ggml_backend_buffer_get_base(buffer),/*.alignment = */ ggml_backend_buffer_get_alignment(buffer),/*.n_free_blocks = */ 0,/*.free_blocks = */ {{0}},- /*.hash_table = */ {{0}},/*.max_size = */ 0,/*.measure = */ false,- /*.parse_seq = */ {0},- /*.parse_seq_len = */ 0,#ifdef GGML_ALLOCATOR_DEBUG/*.allocated_tensors = */ {0},#endif};- ggml_allocr_reset(alloc);+ ggml_tallocr_reset(alloc);return alloc;}-void ggml_allocr_free(struct ggml_allocr * alloc) {+struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t alloc) {+ return alloc->buffer;+}++void ggml_tallocr_free(ggml_tallocr_t alloc) {+ if (alloc == NULL) {+ return;+ }+if (alloc->buffer_owned) {ggml_backend_buffer_free(alloc->buffer);}free(alloc);}-bool ggml_allocr_is_measure(struct ggml_allocr * alloc) {+bool ggml_tallocr_is_measure(ggml_tallocr_t alloc) {return alloc->measure;}-//////////// compute graph allocator+size_t ggml_tallocr_max_size(ggml_tallocr_t alloc) {+ return alloc->max_size;+}++// graph allocator++struct hash_node {+ int n_children;+ int n_views;+};++struct ggml_gallocr {+ ggml_tallocr_t talloc;+ struct ggml_hash_set hash_set;+ struct hash_node * hash_values;+ size_t hash_values_size;+ ggml_tallocr_t * hash_allocs;+ int * parse_seq;+ int parse_seq_len;+};++ggml_gallocr_t ggml_gallocr_new(void) {+ ggml_gallocr_t galloc = (ggml_gallocr_t)malloc(sizeof(struct ggml_gallocr));++ *galloc = (struct ggml_gallocr) {+ /*.talloc = */ NULL,+ /*.hash_set = */ {0},+ /*.hash_values = */ NULL,+ /*.hash_values_size = */ 0,+ /*.hash_allocs = */ NULL,+ /*.parse_seq = */ NULL,+ /*.parse_seq_len = */ 0,+ };++ return galloc;+}++void ggml_gallocr_free(ggml_gallocr_t galloc) {+ if (galloc == NULL) {+ return;+ }++ if (galloc->hash_set.keys != NULL) {+ free(galloc->hash_set.keys);+ }+ if (galloc->hash_values != NULL) {+ free(galloc->hash_values);+ }+ if (galloc->hash_allocs != NULL) {+ free(galloc->hash_allocs);+ }+ if (galloc->parse_seq != NULL) {+ free(galloc->parse_seq);+ }+ free(galloc);+}++void ggml_gallocr_set_parse_seq(ggml_gallocr_t galloc, const int * list, int n) {+ free(galloc->parse_seq);+ galloc->parse_seq = malloc(sizeof(int) * n);++ for (int i = 0; i < n; i++) {+ galloc->parse_seq[i] = list[i];+ }+ galloc->parse_seq_len = n;+}++static struct hash_node * hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {+ size_t i = ggml_hash_find_or_insert(galloc->hash_set, t);+ return &galloc->hash_values[i];+}static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {if (a->type != b->type) {@@ -378,23 +436,39 @@ static bool ggml_op_can_inplace(enum ggml_op op) {}}-static void init_view(struct ggml_allocr * alloc, struct ggml_tensor * view) {- assert(view->view_src != NULL && view->view_src->data != NULL);- view->backend = view->view_src->backend;+static ggml_tallocr_t node_tallocr(ggml_gallocr_t galloc, struct ggml_tensor * node) {+ if (galloc->talloc != NULL) {+ return galloc->talloc;+ }++ return galloc->hash_allocs[ggml_hash_find_or_insert(galloc->hash_set, node)];+}++static void init_view(ggml_gallocr_t galloc, struct ggml_tensor * view, bool update_backend) {+ ggml_tallocr_t alloc = node_tallocr(galloc, view);++ GGML_ASSERT(view->view_src != NULL && view->view_src->data != NULL);+ if (update_backend) {+ view->backend = view->view_src->backend;+ }view->buffer = view->view_src->buffer;view->data = (char *)view->view_src->data + view->view_offs;// FIXME: the view should be initialized by the owning buffer, but currently this breaks the CUDA backend// due to the ggml_tensor_extra_gpu ring buffer overwriting the KV cache extras- assert(ggml_allocr_is_measure(alloc) || !view->buffer || view->buffer->backend == alloc->buffer->backend);- ggml_backend_buffer_init_tensor(alloc->buffer, view);+ assert(ggml_tallocr_is_measure(alloc) || !view->buffer || view->buffer->buft == alloc->buffer->buft);++ if (!alloc->measure) {+ ggml_backend_buffer_init_tensor(alloc->buffer, view);+ }}-static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node) {- struct hash_node * ht = alloc->hash_table;+static void allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {+ ggml_tallocr_t alloc = node_tallocr(galloc, node);+if (node->data == NULL) {if (ggml_is_view(node)) {- init_view(alloc, node);+ init_view(galloc, node, true);} else {// see if we can reuse a parent's buffer (inplace)if (ggml_op_can_inplace(node->op)) {@@ -405,16 +479,16 @@ static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node)}// if the node's data is external, then we cannot re-use it- if (ggml_allocr_is_own(alloc, parent) == false) {+ if (ggml_tallocr_is_own(alloc, parent) == false) {AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);continue;}- struct hash_node * p_hn = hash_get(ht, parent);+ struct hash_node * p_hn = hash_get(galloc, parent);if (parent->data != NULL && p_hn->n_children == 1 && p_hn->n_views == 0 && ggml_are_same_layout(node, parent)) {if (ggml_is_view(parent)) {struct ggml_tensor * view_src = parent->view_src;- struct hash_node * view_src_hn = hash_get(ht, view_src);+ struct hash_node * view_src_hn = hash_get(galloc, view_src);if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {// TODO: the offset of the view parent must be kept to ensure that the op doesn't overwrite// the parent's data that it will need later (same layout requirement). the problem is that then@@ -424,171 +498,307 @@ static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node)AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);node->view_src = view_src;view_src_hn->n_views += 1;- init_view(alloc, node);+ init_view(galloc, node, false);return;}- }- else {+ } else {AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);node->view_src = parent;p_hn->n_views += 1;- init_view(alloc, node);+ init_view(galloc, node, false);return;}}}}- ggml_allocr_alloc(alloc, node);+ ggml_tallocr_alloc(alloc, node);}}}-size_t ggml_allocr_alloc_graph_n(- struct ggml_allocr * alloc,- struct ggml_cgraph ** graphs, int n_graphs,- struct ggml_tensor *** inputs, struct ggml_tensor *** outputs) {+static void free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {+ ggml_tallocr_t alloc = node_tallocr(galloc, node);- // reset hash table- struct hash_node * ht = alloc->hash_table;- memset(ht, 0, sizeof(struct hash_node) * GGML_GRAPH_HASHTABLE_SIZE);+ ggml_tallocr_free_tensor(alloc, node);+}++static void ggml_tallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * gf) {+ const int * parse_seq = galloc->parse_seq;+ int parse_seq_len = galloc->parse_seq_len;// count number of children and views- for (int g = 0; g < n_graphs; g++) {- struct ggml_cgraph * gf = graphs[g];- for (int i = 0; i < gf->n_nodes; i++) {+ for (int i = 0; i < gf->n_nodes; i++) {+ struct ggml_tensor * node = gf->nodes[i];++ if (ggml_is_view(node)) {+ struct ggml_tensor * view_src = node->view_src;+ hash_get(galloc, view_src)->n_views += 1;+ if (node->buffer == NULL && node->data != NULL) {+ // view of a pre-allocated tensor, didn't call init_view() yet+ init_view(galloc, node, true);+ }+ }++ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * parent = node->src[j];+ if (parent == NULL) {+ break;+ }+ hash_get(galloc, parent)->n_children += 1;+ if (ggml_is_view(parent) && parent->buffer == NULL && parent->data != NULL) {+ init_view(galloc, parent, true);+ }+ }+ }++ // allocate tensors+ // if we have parse_seq then we allocate nodes following the list, and we only free nodes at barriers+ int last_barrier_pos = 0;+ int n_nodes = parse_seq_len ? parse_seq_len : gf->n_nodes;++ for (int ind = 0; ind < n_nodes; ind++) {+ // allocate a node if there is no parse_seq or this is not a barrier+ if (parse_seq_len == 0 || parse_seq[ind] != -1) {+ int i = parse_seq_len ? parse_seq[ind] : ind;struct ggml_tensor * node = gf->nodes[i];- if (ggml_is_view(node)) {- struct ggml_tensor * view_src = node->view_src;- hash_get(ht, view_src)->n_views += 1;- if (node->buffer == NULL && node->data != NULL) {- // view of a pre-allocated tensor, didn't call init_view() yet- init_view(alloc, node);+ // allocate parents (leafs)+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * parent = node->src[j];+ if (parent == NULL) {+ break;}+ allocate_node(galloc, parent);}+ // allocate node+ allocate_node(galloc, node);++ AT_PRINTF("exec: %s (%s) <= ", ggml_op_name(node->op), node->name);for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * parent = node->src[j];if (parent == NULL) {break;}- hash_get(ht, parent)->n_children += 1;- if (ggml_is_view(parent) && parent->buffer == NULL && parent->data != NULL) {- init_view(alloc, parent);+ AT_PRINTF("%s", parent->name);+ if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {+ AT_PRINTF(", ");}}+ AT_PRINTF("\n");}- }- // allocate tensors- for (int g = 0; g < n_graphs; g++) {- struct ggml_cgraph * gf = graphs[g];- AT_PRINTF("####### graph %d/%d\n", g, n_graphs);- // graph inputs are allocated first to ensure that they are not overwritten by each other- if (inputs != NULL && inputs[g] != NULL) {- for (int i = 0; inputs[g][i] != NULL; i++) {- struct ggml_tensor * input = inputs[g][i];- AT_PRINTF("input: %s\n", input->name);- allocate_node(alloc, input);- }- }- // if we have parse_seq then we allocate nodes following the list, and we only free nodes at barriers- int last_barrier_pos = 0;- int n_nodes = alloc->parse_seq_len ? alloc->parse_seq_len : gf->n_nodes;-- for (int ind = 0; ind < n_nodes; ind++) {- // allocate a node if there is no parse_seq or this is not a barrier- if ((alloc->parse_seq_len==0) || alloc->parse_seq[ind] != -1) {- int i = alloc->parse_seq_len ? alloc->parse_seq[ind] : ind;- struct ggml_tensor * node = gf->nodes[i];+ // update parents+ // update immediately if there is no parse_seq+ // update only at barriers if there is parse_seq+ if ((parse_seq_len == 0) || parse_seq[ind] == -1) {+ int update_start = parse_seq_len ? last_barrier_pos : ind;+ int update_end = parse_seq_len ? ind : ind + 1;+ for (int i = update_start; i < update_end; i++) {+ int node_i = parse_seq_len ? parse_seq[i] : i;+ struct ggml_tensor * node = gf->nodes[node_i];- // allocate parents (leafs)for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * parent = node->src[j];if (parent == NULL) {break;}- allocate_node(alloc, parent);- }+ struct hash_node * p_hn = hash_get(galloc, parent);+ p_hn->n_children -= 1;- // allocate node- allocate_node(alloc, node);+ //AT_PRINTF("parent %s: %d children, %d views\n", parent->name, parent->n_children, parent->n_views);- AT_PRINTF("exec: %s (%s) <= ", ggml_op_name(node->op), node->name);- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * parent = node->src[j];- if (parent == NULL) {- break;- }- AT_PRINTF("%s", parent->name);- if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {- AT_PRINTF(", ");- }- }- AT_PRINTF("\n");- }-- // update parents- // update immediately if there is no parse_seq- // update only at barriers if there is parse_seq- if ((alloc->parse_seq_len == 0) || alloc->parse_seq[ind] == -1) {- int update_start = alloc->parse_seq_len ? last_barrier_pos : ind;- int update_end = alloc->parse_seq_len ? ind : ind + 1;- for (int i = update_start; i < update_end; i++) {- int node_i = alloc->parse_seq_len ? alloc->parse_seq[i] : i;- struct ggml_tensor * node = gf->nodes[node_i];-- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * parent = node->src[j];- if (parent == NULL) {- break;- }- struct hash_node * p_hn = hash_get(ht, parent);- p_hn->n_children -= 1;-- //AT_PRINTF("parent %s: %d children, %d views\n", parent->name, parent->n_children, parent->n_views);-- if (p_hn->n_children == 0 && p_hn->n_views == 0) {- if (ggml_is_view(parent)) {- struct ggml_tensor * view_src = parent->view_src;- struct hash_node * view_src_hn = hash_get(ht, view_src);- view_src_hn->n_views -= 1;- AT_PRINTF("view_src %s: %d children, %d views\n", view_src->name, view_src_hn->n_children, view_src_hn->n_views);- if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src->data != node->data) {- ggml_allocr_free_tensor(alloc, view_src);- }- }- else {- if (parent->data != node->data) {- ggml_allocr_free_tensor(alloc, parent);- }+ if (p_hn->n_children == 0 && p_hn->n_views == 0) {+ if (ggml_is_view(parent)) {+ struct ggml_tensor * view_src = parent->view_src;+ struct hash_node * view_src_hn = hash_get(galloc, view_src);+ view_src_hn->n_views -= 1;+ AT_PRINTF("view_src %s: %d children, %d views\n", view_src->name, view_src_hn->n_children, view_src_hn->n_views);+ if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0) {+ free_node(galloc, view_src);}}+ else {+ free_node(galloc, parent);+ }}}- AT_PRINTF("\n");- if (alloc->parse_seq_len) {- last_barrier_pos = ind + 1;- }}- }- // free graph outputs here that wouldn't be freed otherwise because they have no children- if (outputs != NULL && outputs[g] != NULL) {- for (int i = 0; outputs[g][i] != NULL; i++) {- struct ggml_tensor * output = outputs[g][i];- AT_PRINTF("output: %s\n", output->name);- ggml_allocr_free_tensor(alloc, output);+ AT_PRINTF("\n");+ if (parse_seq_len) {+ last_barrier_pos = ind + 1;}}}+}- return alloc->max_size;+size_t ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, ggml_tallocr_t talloc, struct ggml_cgraph * graph) {+ size_t hash_size = graph->visited_hash_table.size;++ // check if the hash table is initialized and large enough+ if (galloc->hash_set.size < hash_size) {+ if (galloc->hash_set.keys != NULL) {+ free(galloc->hash_set.keys);+ }+ if (galloc->hash_values != NULL) {+ free(galloc->hash_values);+ }+ galloc->hash_set.keys = malloc(sizeof(struct ggml_tensor *) * hash_size);+ galloc->hash_set.size = hash_size;+ galloc->hash_values = malloc(sizeof(struct hash_node) * hash_size);+ }++ // reset hash table+ memset(galloc->hash_set.keys, 0, sizeof(struct ggml_tensor *) * hash_size);+ memset(galloc->hash_values, 0, sizeof(struct hash_node) * hash_size);++ galloc->talloc = talloc;+ ggml_tallocr_alloc_graph_impl(galloc, graph);+ galloc->talloc = NULL;++ size_t max_size = ggml_tallocr_max_size(talloc);++ return max_size;}-size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph) {- return ggml_allocr_alloc_graph_n(alloc, &graph, 1, NULL, NULL);+void ggml_gallocr_alloc_graph_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, struct ggml_hash_set hash_set, ggml_tallocr_t * hash_node_talloc) {+ const size_t hash_size = hash_set.size;++ GGML_ASSERT(hash_size >= (size_t)(graph->n_nodes + graph->n_leafs));++ galloc->talloc = NULL;++ // alloc hash_values if needed+ if (galloc->hash_values == NULL || galloc->hash_values_size < hash_size) {+ free(galloc->hash_values);+ galloc->hash_values = malloc(sizeof(struct hash_node) * hash_size);+ galloc->hash_values_size = hash_size;+ }++ // free hash_set.keys if needed+ if (galloc->hash_set.keys != NULL) {+ free(galloc->hash_set.keys);+ }+ galloc->hash_set = hash_set;++ // reset hash values+ memset(galloc->hash_values, 0, sizeof(struct hash_node) * hash_size);++ galloc->hash_allocs = hash_node_talloc;++ ggml_tallocr_alloc_graph_impl(galloc, graph);++ // remove unowned resources+ galloc->hash_set.keys = NULL;+ galloc->hash_allocs = NULL;}-size_t ggml_allocr_max_size(struct ggml_allocr * alloc) {- return alloc->max_size;+// legacy API wrapper++struct ggml_allocr {+ ggml_tallocr_t talloc;+ ggml_gallocr_t galloc;+};++static ggml_allocr_t ggml_allocr_new_impl(ggml_tallocr_t talloc) {+ ggml_allocr_t alloc = (ggml_allocr_t)malloc(sizeof(struct ggml_allocr));+ *alloc = (struct ggml_allocr) {+ /*.talloc = */ talloc,+ /*.galloc = */ ggml_gallocr_new(),+ };+ return alloc;+}++ggml_allocr_t ggml_allocr_new(void * data, size_t size, size_t alignment) {+ return ggml_allocr_new_impl(ggml_tallocr_new(data, size, alignment));+}++ggml_allocr_t ggml_allocr_new_measure(size_t alignment) {+ return ggml_allocr_new_impl(ggml_tallocr_new_measure(alignment));+}++ggml_allocr_t ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer) {+ return ggml_allocr_new_impl(ggml_tallocr_new_from_buffer(buffer));+}++ggml_allocr_t ggml_allocr_new_from_backend(struct ggml_backend * backend, size_t size) {+ return ggml_allocr_new_impl(ggml_tallocr_new_from_backend(backend, size));+}++ggml_allocr_t ggml_allocr_new_measure_from_backend(struct ggml_backend * backend) {+ return ggml_allocr_new_impl(ggml_tallocr_new_measure_from_backend(backend));+}++struct ggml_backend_buffer * ggml_allocr_get_buffer(ggml_allocr_t alloc) {+ return ggml_tallocr_get_buffer(alloc->talloc);+}++void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n) {+ ggml_gallocr_set_parse_seq(alloc->galloc, list, n);+}++void ggml_allocr_free(ggml_allocr_t alloc) {+ ggml_gallocr_free(alloc->galloc);+ ggml_tallocr_free(alloc->talloc);+ free(alloc);+}++bool ggml_allocr_is_measure(ggml_allocr_t alloc) {+ return ggml_tallocr_is_measure(alloc->talloc);+}++void ggml_allocr_reset(ggml_allocr_t alloc) {+ ggml_tallocr_reset(alloc->talloc);+}++void ggml_allocr_alloc(ggml_allocr_t alloc, struct ggml_tensor * tensor) {+ ggml_tallocr_alloc(alloc->talloc, tensor);+}++size_t ggml_allocr_max_size(ggml_allocr_t alloc) {+ return ggml_tallocr_max_size(alloc->talloc);+}++size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph * graph) {+ return ggml_gallocr_alloc_graph(alloc->galloc, alloc->talloc, graph);+}++// utils+ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {+ GGML_ASSERT(ggml_get_no_alloc(ctx) == true);++ size_t alignment = ggml_backend_buft_get_alignment(buft);++ size_t nbytes = 0;+ for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {+ if (t->data == NULL && t->view_src == NULL) {+ nbytes += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);+ }+ }++ if (nbytes == 0) {+ fprintf(stderr, "%s: no tensors to allocate\n", __func__);+ return NULL;+ }++ ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, nbytes);+ ggml_tallocr_t tallocr = ggml_tallocr_new_from_buffer(buffer);++ for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {+ if (t->data == NULL) {+ if (t->view_src == NULL) {+ ggml_tallocr_alloc(tallocr, t);+ } else {+ ggml_backend_view_init(buffer, t);+ }+ }+ }++ ggml_tallocr_free(tallocr);++ return buffer;+}++ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) {+ return ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_get_default_buffer_type(backend));}diff --git a/llama.cpp/ggml-alloc.h b/llama.cpp/ggml-alloc.hindex e3875887..ad87cebc 100644--- a/llama.cpp/ggml-alloc.h+++ b/llama.cpp/ggml-alloc.h@@ -6,27 +6,86 @@extern "C" {#endif+struct ggml_backend;struct ggml_backend_buffer;+struct ggml_backend_buffer_type;-GGML_API struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment);-GGML_API struct ggml_allocr * ggml_allocr_new_measure(size_t alignment);-GGML_API struct ggml_allocr * ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer);+//+// Legacy API+//++typedef struct ggml_allocr * ggml_allocr_t;++// initialize allocator for use with CPU backend only+GGML_API ggml_allocr_t ggml_allocr_new(void * data, size_t size, size_t alignment);+GGML_API ggml_allocr_t ggml_allocr_new_measure(size_t alignment);++// initialize allocator for use with ggml-backend+GGML_API ggml_allocr_t ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer);+GGML_API ggml_allocr_t ggml_allocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer+GGML_API ggml_allocr_t ggml_allocr_new_measure_from_backend(struct ggml_backend * backend);++GGML_API struct ggml_backend_buffer * ggml_allocr_get_buffer(ggml_allocr_t alloc);// tell the allocator to parse nodes following the order described in the list// you should call this if your graph are optimized to execute out-of-order-GGML_API void ggml_allocr_set_parse_seq(struct ggml_allocr * alloc, const int * list, int n);--GGML_API void ggml_allocr_free (struct ggml_allocr * alloc);-GGML_API bool ggml_allocr_is_measure (struct ggml_allocr * alloc);-GGML_API void ggml_allocr_reset (struct ggml_allocr * alloc);-GGML_API void ggml_allocr_alloc (struct ggml_allocr * alloc, struct ggml_tensor * tensor);-GGML_API size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph);-GGML_API size_t ggml_allocr_max_size (struct ggml_allocr * alloc);--GGML_API size_t ggml_allocr_alloc_graph_n(- struct ggml_allocr * alloc,- struct ggml_cgraph ** graphs, int n_graphs,- struct ggml_tensor *** inputs, struct ggml_tensor *** outputs);+GGML_API void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n);++GGML_API void ggml_allocr_free (ggml_allocr_t alloc);+GGML_API bool ggml_allocr_is_measure (ggml_allocr_t alloc);+GGML_API void ggml_allocr_reset (ggml_allocr_t alloc);+GGML_API void ggml_allocr_alloc (ggml_allocr_t alloc, struct ggml_tensor * tensor);+GGML_API size_t ggml_allocr_max_size (ggml_allocr_t alloc);++GGML_API size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph * graph);++//+// ggml-backend v2 API+//++// Seperate tensor and graph allocator objects+// This is necessary for multi-backend allocation because the graph allocator needs to use multiple tensor allocators+// The original API is kept as a wrapper around the new API++// Tensor allocator+typedef struct ggml_tallocr * ggml_tallocr_t;++GGML_API ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment);+GGML_API ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment);+GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer);+GGML_API ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer+GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend);++GGML_API struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t talloc);++GGML_API void ggml_tallocr_free (ggml_tallocr_t talloc);+GGML_API bool ggml_tallocr_is_measure (ggml_tallocr_t talloc);+GGML_API void ggml_tallocr_reset (ggml_tallocr_t talloc);+GGML_API void ggml_tallocr_alloc (ggml_tallocr_t talloc, struct ggml_tensor * tensor);+GGML_API size_t ggml_tallocr_max_size (ggml_tallocr_t talloc);+++// Graph allocator+typedef struct ggml_gallocr * ggml_gallocr_t;++GGML_API ggml_gallocr_t ggml_gallocr_new(void);+GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);++GGML_API void ggml_gallocr_set_parse_seq(ggml_gallocr_t galloc, const int * list, int n);+GGML_API size_t ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, ggml_tallocr_t talloc, struct ggml_cgraph * graph);++// Allocate tensors from the allocators given by the hash table+GGML_API void ggml_gallocr_alloc_graph_n(+ ggml_gallocr_t galloc,+ struct ggml_cgraph * graph,+ struct ggml_hash_set hash_set,+ ggml_tallocr_t * hash_node_talloc);+++// Utils+// Create a buffer and allocate all the tensors in a ggml_context+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, struct ggml_backend_buffer_type * buft);+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, struct ggml_backend * backend);#ifdef __cplusplus}diff --git a/llama.cpp/ggml-backend-impl.h b/llama.cpp/ggml-backend-impl.hnew file mode 100644index 00000000..f588af60--- /dev/null+++ b/llama.cpp/ggml-backend-impl.h@@ -0,0 +1,112 @@+#pragma once++// ggml-backend internal header++#include "ggml-backend.h"++#ifdef __cplusplus+extern "C" {+#endif++ //+ // Backend buffer+ //++ // buffer type+ typedef void * ggml_backend_buffer_type_context_t;++ struct ggml_backend_buffer_type_i {+ ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);+ size_t (*get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment+ size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding+ bool (*supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend+ };++ struct ggml_backend_buffer_type {+ struct ggml_backend_buffer_type_i iface;+ ggml_backend_buffer_type_context_t context;+ };++ // buffer+ typedef void * ggml_backend_buffer_context_t;++ struct ggml_backend_buffer_i {+ void (*free_buffer)(ggml_backend_buffer_t buffer);+ //void (*reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras+ void * (*get_base) (ggml_backend_buffer_t buffer);+ void (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ // (optional) copy tensor between different buffer-type, allow for single-copy tranfers+ void (*cpy_tensor_from)(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);+ void (*cpy_tensor_to) (ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);+ };++ struct ggml_backend_buffer {+ struct ggml_backend_buffer_i iface;+ ggml_backend_buffer_type_t buft;+ ggml_backend_buffer_context_t context;+ size_t size;+ };++ ggml_backend_buffer_t ggml_backend_buffer_init(+ ggml_backend_buffer_type_t buft,+ struct ggml_backend_buffer_i iface,+ ggml_backend_buffer_context_t context,+ size_t size);+++ //+ // Backend+ //++ typedef void * ggml_backend_context_t;++ struct ggml_backend_i {+ const char * (*get_name)(ggml_backend_t backend);++ void (*free)(ggml_backend_t backend);++ // buffer allocation+ ggml_backend_buffer_type_t (*get_default_buffer_type)(ggml_backend_t backend);++ // (optional) asynchroneous tensor data access+ void (*set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);++ // (optional) asynchroneous tensor copy+ void (*cpy_tensor_from_async)(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);+ void (*cpy_tensor_to_async) (ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);++ void (*synchronize) (ggml_backend_t backend);++ // compute graph with a plan+ ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, struct ggml_cgraph * cgraph);+ void (*graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);+ void (*graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);++ // compute graph without a plan+ void (*graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph);++ // check if the backend supports an operation+ bool (*supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);+ };++ struct ggml_backend {+ struct ggml_backend_i iface;++ ggml_backend_context_t context;+ };+++ //+ // Backend registry+ //++ typedef ggml_backend_t (*ggml_backend_init_fn)(const char * params, void * user_data);++ void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);++#ifdef __cplusplus+}+#endifdiff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex ca8d83da..a436589d 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -1,20 +1,47 @@-#include "ggml-backend.h"+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "ggml-backend-impl.h"#include "ggml-alloc.h"+#include "llama.cpp/ggml-cuda.h"+#include "ggml-impl.h"#include <assert.h>+#include <limits.h>#include <stdarg.h>#include <stdio.h>#include <stdlib.h>#include <string.h>-#define UNUSED GGML_UNUSED#define MAX(a, b) ((a) > (b) ? (a) : (b))++// backend buffer type++ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ return buft->iface.alloc_buffer(buft, size);+}++size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {+ return buft->iface.get_alignment(buft);+}++size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) {+ // get_alloc_size is optional, defaults to ggml_nbytes+ if (buft->iface.get_alloc_size) {+ return buft->iface.get_alloc_size(buft, tensor);+ }+ return ggml_nbytes(tensor);+}++bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ return buft->iface.supports_backend(buft, backend);+}+// backend bufferggml_backend_buffer_t ggml_backend_buffer_init(- struct ggml_backend * backend,+ ggml_backend_buffer_type_t buft,struct ggml_backend_buffer_i iface,ggml_backend_buffer_context_t context,size_t size) {@@ -24,7 +51,7 @@ ggml_backend_buffer_t ggml_backend_buffer_init((*buffer) = (struct ggml_backend_buffer) {/* .interface = */ iface,- /* .backend = */ backend,+ /* .buft = */ buft,/* .context = */ context,/* .size = */ size,};@@ -33,84 +60,111 @@ ggml_backend_buffer_t ggml_backend_buffer_init(}void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {+ if (buffer == NULL) {+ return;+ }+if (buffer->iface.free_buffer != NULL) {buffer->iface.free_buffer(buffer);}free(buffer);}-size_t ggml_backend_buffer_get_alignment(ggml_backend_buffer_t buffer) {- return ggml_backend_get_alignment(buffer->backend);+size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {+ return buffer->size;}void * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) {- return buffer->iface.get_base(buffer);-}+ void * base = buffer->iface.get_base(buffer);-size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {- return buffer->size;-}+ GGML_ASSERT(base != NULL && "backend buffer base cannot be NULL");-size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {- if (buffer->iface.get_alloc_size) {- return buffer->iface.get_alloc_size(buffer, tensor);- }- return ggml_nbytes(tensor);+ return base;}void ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {+ // init_tensor is optionalif (buffer->iface.init_tensor) {buffer->iface.init_tensor(buffer, tensor);}}-void ggml_backend_buffer_free_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {- if (buffer->iface.free_tensor) {- buffer->iface.free_tensor(buffer, tensor);- }+size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer) {+ return ggml_backend_buft_get_alignment(ggml_backend_buffer_type(buffer));}-// backend+size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {+ return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type(buffer), tensor);+}-ggml_backend_t ggml_get_backend(const struct ggml_tensor * tensor) {- return tensor->buffer->backend;+ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer) {+ return buffer->buft;}+// backend+const char * ggml_backend_name(ggml_backend_t backend) {+ if (backend == NULL) {+ return "NULL";+ }return backend->iface.get_name(backend);}void ggml_backend_free(ggml_backend_t backend) {+ if (backend == NULL) {+ return;+ }+backend->iface.free(backend);}+ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend) {+ return backend->iface.get_default_buffer_type(backend);+}+ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size) {- return backend->iface.alloc_buffer(backend, size);+ return ggml_backend_buft_alloc_buffer(ggml_backend_get_default_buffer_type(backend), size);}size_t ggml_backend_get_alignment(ggml_backend_t backend) {- return backend->iface.get_alignment(backend);+ return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));}-void ggml_backend_tensor_set_async(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- ggml_get_backend(tensor)->iface.set_tensor_async(ggml_get_backend(tensor), tensor, data, offset, size);+void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");++ backend->iface.set_tensor_async(backend, tensor, data, offset, size);}-void ggml_backend_tensor_get_async(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {- ggml_get_backend(tensor)->iface.get_tensor_async(ggml_get_backend(tensor), tensor, data, offset, size);+void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");++ backend->iface.get_tensor_async(backend, tensor, data, offset, size);}void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- ggml_get_backend(tensor)->iface.set_tensor_async(ggml_get_backend(tensor), tensor, data, offset, size);- ggml_get_backend(tensor)->iface.synchronize(ggml_get_backend(tensor));+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set");+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");++ tensor->buffer->iface.set_tensor(tensor->buffer, tensor, data, offset, size);}void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {- ggml_get_backend(tensor)->iface.get_tensor_async(ggml_get_backend(tensor), tensor, data, offset, size);- ggml_get_backend(tensor)->iface.synchronize(ggml_get_backend(tensor));+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set");+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");++ tensor->buffer->iface.get_tensor(tensor->buffer, tensor, data, offset, size);}void ggml_backend_synchronize(ggml_backend_t backend) {+ if (backend->iface.synchronize == NULL) {+ return;+ }+backend->iface.synchronize(backend);}@@ -124,10 +178,16 @@ void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plavoid ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {backend->iface.graph_plan_compute(backend, plan);++ // TODO: optional sync+ ggml_backend_synchronize(backend);}void ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {backend->iface.graph_compute(backend, cgraph);++ // TODO: optional sync+ ggml_backend_synchronize(backend);}bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {@@ -156,7 +216,7 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst//printf("dst: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", dst->name, (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], (int)dst->nb[0], (int)dst->nb[1], (int)dst->nb[2], (int)dst->nb[3]);GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");- // printf("cpy tensor %s from %s to %s (%lu bytes)\n", src->name, ggml_backend_name(src->backend), ggml_backend_name(dst->backend), ggml_nbytes(src));+ // fprintf(stderr, "cpy tensor %s from %s to %s (%lu bytes)\n", src->name, ggml_backend_name(src->backend), ggml_backend_name(dst->backend), ggml_nbytes(src));if (src == dst) {return;@@ -164,14 +224,15 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst// TODO: allow backends to support copy to/from same backend- if (ggml_get_backend(dst)->iface.cpy_tensor_from != NULL) {- ggml_get_backend(dst)->iface.cpy_tensor_from(ggml_get_backend(dst)->context, src, dst);- } else if (ggml_get_backend(src)->iface.cpy_tensor_to != NULL) {- ggml_get_backend(src)->iface.cpy_tensor_to(ggml_get_backend(src)->context, src, dst);+ if (dst->buffer->iface.cpy_tensor_from != NULL) {+ dst->buffer->iface.cpy_tensor_from(dst->buffer, src, dst);+ } else if (src->buffer->iface.cpy_tensor_to != NULL) {+ src->buffer->iface.cpy_tensor_to(src->buffer, src, dst);} else {// shouldn't be hit when copying from/to CPU#ifndef NDEBUG- fprintf(stderr, "ggml_backend_tensor_copy: neither cpy_tensor_from nor cpy_tensor_to are implemented for backends %s and %s, falling back to get/set\n", ggml_backend_name(src->buffer->backend), ggml_backend_name(dst->buffer->backend));+ fprintf(stderr, "ggml_backend_tensor_copy: neither cpy_tensor_from nor cpy_tensor_to "+ "are implemented for %s and %s, falling back to get/set\n", src->name, dst->name);#endifsize_t nbytes = ggml_nbytes(src);void * data = malloc(nbytes);@@ -181,100 +242,270 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst}}-// backend CPU+// backend registry-struct ggml_backend_cpu_context {- int n_threads;- void * work_data;- size_t work_size;+#define GGML_MAX_BACKENDS_REG 16++struct ggml_backend_reg {+ char name[128];+ ggml_backend_init_fn init_fn;+ ggml_backend_buffer_type_t default_buffer_type;+ void * user_data;};-static const char * ggml_backend_cpu_name(ggml_backend_t backend) {- return "CPU";+static struct ggml_backend_reg ggml_backend_registry[GGML_MAX_BACKENDS_REG];+static size_t ggml_backend_registry_count = 0;- UNUSED(backend);+static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data);++int ggml_backend_cuda_reg_devices() {+ int device_count = ggml_cuda_get_device_count();+ //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization+ for (int i = 0; i < device_count; i++) {+ char name[128];+ snprintf(name, sizeof(name), "%s%d", GGML_CUDA_NAME, i);+ ggml_backend_register(name, ggml_backend_reg_cuda_init, ggml_backend_cuda_buffer_type(i), (void *) (intptr_t) i);+ }+ return device_count;}-static void ggml_backend_cpu_free(ggml_backend_t backend) {- struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;- free(cpu_ctx->work_data);- free(cpu_ctx);- free(backend);+static void ggml_backend_registry_init(void) {+ static bool initialized = false;++ if (initialized) {+ return;+ }++ initialized = true;++ ggml_backend_register("CPU", ggml_backend_reg_cpu_init, ggml_backend_cpu_buffer_type(), NULL);++ // add forward decls here to avoid including the backend headers+#ifdef GGML_USE_CUBLAS+ extern void ggml_backend_cuda_reg_devices(void);+ ggml_backend_cuda_reg_devices();+#endif++#ifdef GGML_USE_METAL+ extern ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data);+ extern ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);+ ggml_backend_register("Metal", ggml_backend_reg_metal_init, ggml_backend_metal_buffer_type(), NULL);+#endif+}++void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {+ GGML_ASSERT(ggml_backend_registry_count < GGML_MAX_BACKENDS_REG);++ int id = ggml_backend_registry_count;++ ggml_backend_registry[id] = (struct ggml_backend_reg) {+ /* .name = */ {0},+ /* .fn = */ init_fn,+ /* .default_buffer_type = */ default_buffer_type,+ /* .user_data = */ user_data,+ };++ snprintf(ggml_backend_registry[id].name, sizeof(ggml_backend_registry[id].name), "%s", name);++#ifndef NDEBUG+ fprintf(stderr, "%s: registered backend %s\n", __func__, name);+#endif++ ggml_backend_registry_count++;+}++size_t ggml_backend_reg_get_count(void) {+ ggml_backend_registry_init();++ return ggml_backend_registry_count;+}++size_t ggml_backend_reg_find_by_name(const char * name) {+ ggml_backend_registry_init();++ for (size_t i = 0; i < ggml_backend_registry_count; i++) {+ // TODO: case insensitive in a portable way+ if (strcmp(ggml_backend_registry[i].name, name) == 0) {+ return i;+ }+ }+ return SIZE_MAX;+}++// init from backend:params string+ggml_backend_t ggml_backend_reg_init_backend_from_str(const char * backend_str) {+ ggml_backend_registry_init();++ const char * params = strchr(backend_str, ':');+ char backend_name[128];+ if (params == NULL) {+ strcpy(backend_name, backend_str);+ params = "";+ } else {+ strncpy(backend_name, backend_str, params - backend_str);+ backend_name[params - backend_str] = '\0';+ params++;+ }++ size_t backend_i = ggml_backend_reg_find_by_name(backend_name);+ if (backend_i == SIZE_MAX) {+ fprintf(stderr, "%s: backend %s not found\n", __func__, backend_name);+ return NULL;+ }++ return ggml_backend_reg_init_backend(backend_i, params);+}++const char * ggml_backend_reg_get_name(size_t i) {+ ggml_backend_registry_init();++ GGML_ASSERT(i < ggml_backend_registry_count);+ return ggml_backend_registry[i].name;+}++ggml_backend_t ggml_backend_reg_init_backend(size_t i, const char * params) {+ ggml_backend_registry_init();++ GGML_ASSERT(i < ggml_backend_registry_count);+ return ggml_backend_registry[i].init_fn(params, ggml_backend_registry[i].user_data);+}++ggml_backend_buffer_type_t ggml_backend_reg_get_default_buffer_type(size_t i) {+ ggml_backend_registry_init();++ GGML_ASSERT(i < ggml_backend_registry_count);+ return ggml_backend_registry[i].default_buffer_type;+}++ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size) {+ ggml_backend_registry_init();++ GGML_ASSERT(i < ggml_backend_registry_count);+ return ggml_backend_buft_alloc_buffer(ggml_backend_registry[i].default_buffer_type, size);}+// backend CPU+static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {return (void *)buffer->context;}static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {free(buffer->context);- UNUSED(buffer);+ GGML_UNUSED(buffer);+}++static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");++ memcpy((char *)tensor->data + offset, data, size);++ GGML_UNUSED(buffer);+}++static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");++ memcpy(data, (const char *)tensor->data + offset, size);++ GGML_UNUSED(buffer);+}++static void ggml_backend_cpu_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {+ ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));++ GGML_UNUSED(buffer);+}++static void ggml_backend_cpu_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {+ ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));++ GGML_UNUSED(buffer);}static struct ggml_backend_buffer_i cpu_backend_buffer_i = {- /* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,- /* .get_base = */ ggml_backend_cpu_buffer_get_base,- /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes- /* .init_tensor = */ NULL, // no initialization required- /* .free_tensor = */ NULL, // no cleanup required+ /* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,+ /* .init_tensor = */ NULL, // no initialization required+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,+ /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,+ /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,};// for buffers from ptr, free is not calledstatic struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = {- /* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed- /* .get_base = */ ggml_backend_cpu_buffer_get_base,- /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes- /* .init_tensor = */ NULL,- /* .free_tensor = */ NULL,+ /* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,+ /* .init_tensor = */ NULL, // no initialization required+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,+ /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,+ /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,};static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512-static ggml_backend_buffer_t ggml_backend_cpu_alloc_buffer(ggml_backend_t backend, size_t size) {+static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {size += TENSOR_ALIGNMENT; // malloc may return an address that is not alignedvoid * data = malloc(size); // TODO: maybe use GGML_ALIGNED_MALLOC?- return ggml_backend_buffer_init(backend, cpu_backend_buffer_i, data, size);+ GGML_ASSERT(data != NULL && "failed to allocate buffer");++ return ggml_backend_buffer_init(buft, cpu_backend_buffer_i, data, size);}-static size_t ggml_backend_cpu_get_alignment(ggml_backend_t backend) {+static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {return TENSOR_ALIGNMENT;- UNUSED(backend);-}-static void ggml_backend_cpu_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_UNUSED(buft);+}- memcpy((char *)tensor->data + offset, data, size);+static bool ggml_backend_cpu_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ return ggml_backend_is_cpu(backend);- UNUSED(backend);+ GGML_UNUSED(buft);}-static void ggml_backend_cpu_get_tensor_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");-- memcpy(data, (const char *)tensor->data + offset, size);+ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {+ static struct ggml_backend_buffer_type ggml_backend_buffer_type_cpu = {+ /* .iface = */ {+ /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes+ /* .supports_backend = */ ggml_backend_cpu_buffer_type_supports_backend,+ },+ /* .context = */ NULL,+ };- UNUSED(backend);+ return &ggml_backend_buffer_type_cpu;}-static void ggml_backend_cpu_synchronize(ggml_backend_t backend) {- UNUSED(backend);-}+struct ggml_backend_cpu_context {+ int n_threads;+ void * work_data;+ size_t work_size;+};-static void ggml_backend_cpu_cpy_tensor_from(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) {- ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));+static const char * ggml_backend_cpu_name(ggml_backend_t backend) {+ return "CPU";- UNUSED(backend);+ GGML_UNUSED(backend);}-static void ggml_backend_cpu_cpy_tensor_to(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) {- // for a backend such as CUDA that can queue async calls, it is ok to do this asynchronously, but it may not be the case for other backends- ggml_backend_tensor_set_async(dst, src->data, 0, ggml_nbytes(src));+static void ggml_backend_cpu_free(ggml_backend_t backend) {+ struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;+ free(cpu_ctx->work_data);+ free(cpu_ctx);+ free(backend);+}- UNUSED(backend);+static ggml_backend_buffer_type_t ggml_backend_cpu_get_default_buffer_type(ggml_backend_t backend) {+ return ggml_backend_cpu_buffer_type();++ GGML_UNUSED(backend);}struct ggml_backend_plan_cpu {@@ -303,7 +534,7 @@ static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backenfree(cpu_plan->cplan.work_data);free(cpu_plan);- UNUSED(backend);+ GGML_UNUSED(backend);}static void ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {@@ -311,7 +542,7 @@ static void ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_bacggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);- UNUSED(backend);+ GGML_UNUSED(backend);}static void ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {@@ -332,25 +563,25 @@ static void ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cstatic bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {return true;- UNUSED(backend);- UNUSED(op);++ GGML_UNUSED(backend);+ GGML_UNUSED(op);}static struct ggml_backend_i cpu_backend_i = {- /* .get_name = */ ggml_backend_cpu_name,- /* .free = */ ggml_backend_cpu_free,- /* .alloc_buffer = */ ggml_backend_cpu_alloc_buffer,- /* .get_alignment = */ ggml_backend_cpu_get_alignment,- /* .set_tensor_async = */ ggml_backend_cpu_set_tensor_async,- /* .get_tensor_async = */ ggml_backend_cpu_get_tensor_async,- /* .synchronize = */ ggml_backend_cpu_synchronize,- /* .cpy_tensor_from = */ ggml_backend_cpu_cpy_tensor_from,- /* .cpy_tensor_to = */ ggml_backend_cpu_cpy_tensor_to,- /* .graph_plan_create = */ ggml_backend_cpu_graph_plan_create,- /* .graph_plan_free = */ ggml_backend_cpu_graph_plan_free,- /* .graph_plan_compute = */ ggml_backend_cpu_graph_plan_compute,- /* .graph_compute = */ ggml_backend_cpu_graph_compute,- /* .supports_op = */ ggml_backend_cpu_supports_op,+ /* .get_name = */ ggml_backend_cpu_name,+ /* .free = */ ggml_backend_cpu_free,+ /* .get_default_buffer_type = */ ggml_backend_cpu_get_default_buffer_type,+ /* .set_tensor_async = */ NULL,+ /* .get_tensor_async = */ NULL,+ /* .cpy_tensor_from_async = */ NULL,+ /* .cpy_tensor_to_async = */ NULL,+ /* .synchronize = */ NULL,+ /* .graph_plan_create = */ ggml_backend_cpu_graph_plan_create,+ /* .graph_plan_free = */ ggml_backend_cpu_graph_plan_free,+ /* .graph_plan_compute = */ ggml_backend_cpu_graph_plan_compute,+ /* .graph_compute = */ ggml_backend_cpu_graph_compute,+ /* .supports_op = */ ggml_backend_cpu_supports_op,};ggml_backend_t ggml_backend_cpu_init(void) {@@ -380,6 +611,761 @@ void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {ctx->n_threads = n_threads;}-ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(ggml_backend_t backend_cpu, void * ptr, size_t size) {- return ggml_backend_buffer_init(backend_cpu, cpu_backend_buffer_i_from_ptr, ptr, size);+ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {+ return ggml_backend_buffer_init(ggml_backend_cpu_buffer_type(), cpu_backend_buffer_i_from_ptr, ptr, size);+}++static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data) {+ return ggml_backend_cpu_init();++ GGML_UNUSED(params);+ GGML_UNUSED(user_data);+}+++// scheduler++#define GGML_MAX_BACKENDS 4+#define GGML_MAX_SPLITS 256+#define GGML_MAX_SPLIT_INPUTS 16++struct ggml_backend_sched_split {+ ggml_tallocr_t tallocr;+ int i_start;+ int i_end;+ struct ggml_tensor * inputs[GGML_MAX_SPLIT_INPUTS];+ int n_inputs;+ struct ggml_cgraph graph;+};++struct ggml_backend_sched {+ int n_backends;+ ggml_backend_t backends[GGML_MAX_BACKENDS];+ ggml_tallocr_t tallocs[GGML_MAX_BACKENDS];++ ggml_gallocr_t galloc;++ struct ggml_hash_set hash_set;+ ggml_tallocr_t * node_talloc; // [hash_set.size]+ struct ggml_tensor * (* node_copies)[GGML_MAX_BACKENDS]; // [hash_set.size][GGML_MAX_BACKENDS]++ struct ggml_cgraph * graph;+ struct ggml_backend_sched_split splits[GGML_MAX_SPLITS];+ int n_splits;++ struct ggml_context * ctx;++ // align context_buffer to GGML_MEM_ALIGN+ #ifdef _MSC_VER+ __declspec(align(GGML_MEM_ALIGN))+ #else+ __attribute__((aligned(GGML_MEM_ALIGN)))+ #endif+ char context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)];+};++#define hash_id(node) ggml_hash_find_or_insert(sched->hash_set, node)+#define node_allocr(node) sched->node_talloc[hash_id(node)]++static bool ggml_is_view_op(enum ggml_op op) {+ return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;+}++// returns the priority of the backend, lower is better+static int sched_backend_prio(ggml_backend_sched_t sched, ggml_backend_t backend) {+ for (int i = 0; i < sched->n_backends; i++) {+ if (sched->backends[i] == backend) {+ return i;+ }+ }+ return INT_MAX;+}++static int sched_allocr_prio(ggml_backend_sched_t sched, ggml_tallocr_t allocr) {+ for (int i = 0; i < sched->n_backends; i++) {+ if (sched->tallocs[i] == allocr) {+ return i;+ }+ }+ return INT_MAX;+}++static ggml_backend_t get_buffer_backend(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) {+ if (buffer == NULL) {+ return NULL;+ }+ // find highest prio backend that supports the buffer type+ for (int i = 0; i < sched->n_backends; i++) {+ if (ggml_backend_buft_supports_backend(buffer->buft, sched->backends[i])) {+ return sched->backends[i];+ }+ }+ GGML_ASSERT(false && "tensor buffer type not supported by any backend");+}++static ggml_backend_t get_allocr_backend(ggml_backend_sched_t sched, ggml_tallocr_t allocr) {+ if (allocr == NULL) {+ return NULL;+ }+ // find highest prio backend that supports the buffer type+ for (int i = 0; i < sched->n_backends; i++) {+ if (sched->tallocs[i] == allocr) {+ return sched->backends[i];+ }+ }+ GGML_UNREACHABLE();+}++#if 0+static char causes[GGML_DEFAULT_GRAPH_SIZE*8 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug, remove+#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)+#define GET_CAUSE(node) causes[hash_id(node)]+#else+#define SET_CAUSE(node, ...)+#define GET_CAUSE(node) ""+#endif++// returns the backend that should be used for the node based on the current locations+static ggml_backend_t sched_backend_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * node) {+ // if the dst tensor is already allocated in a buffer, we must assume that it is critical to keep it there+ // ie. kv cache updates+ // note that this doesn't allow fallback to CPU. need to add output tensors to the splits to copy the data back to the original backend.+ // dst+ ggml_backend_t cur_backend = get_buffer_backend(sched, node->buffer);+ if (cur_backend != NULL) {+ SET_CAUSE(node, "1.dst");+ return cur_backend;+ }++ // view_src+ if (node->view_src != NULL && get_buffer_backend(sched, node->view_src->buffer) != NULL) {+ SET_CAUSE(node, "1.vsrc");+ return get_buffer_backend(sched, node->view_src->buffer);+ }++ // src+ int cur_prio = INT_MAX;+ size_t cur_size = 0;++ for (int i = 0; i < GGML_MAX_SRC; i++) {+ const struct ggml_tensor * src = node->src[i];+ if (src == NULL) {+ break;+ }+ ggml_backend_t src_backend = get_buffer_backend(sched, src->buffer);+ if (src_backend != NULL) {+ int src_prio = sched_backend_prio(sched, src_backend);+ size_t src_size = ggml_nbytes(src);+ if (src_prio < cur_prio && src_size >= cur_size) {+ cur_prio = src_prio;+ cur_size = src_size;+ cur_backend = src_backend;+ SET_CAUSE(node, "1.src%d", i);+ }+ }+ }+ return cur_backend;+}++static char * fmt_size(size_t size) {+ static char buffer[128];+ if (size >= 1024*1024) {+ sprintf(buffer, "%zuM", size/1024/1024);+ } else {+ sprintf(buffer, "%zuK", size/1024);+ }+ return buffer;+}++static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+ int cur_split = 0;+ for (int i = 0; i < graph->n_nodes; i++) {+ if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {+ ggml_backend_t split_backend = get_allocr_backend(sched, sched->splits[cur_split].tallocr);+ fprintf(stderr, "\n## SPLIT #%d: %s # %d inputs: ", cur_split, ggml_backend_name(split_backend),+ sched->splits[cur_split].n_inputs);+ for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {+ fprintf(stderr, "[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name,+ fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j])));+ }+ fprintf(stderr, "\n");+ cur_split++;+ }+ struct ggml_tensor * node = graph->nodes[i];+ if (ggml_is_view_op(node->op)) {+ continue;+ }+ ggml_tallocr_t node_allocr = node_allocr(node);+ ggml_backend_t node_backend = node_allocr ? get_allocr_backend(sched, node_allocr) : NULL; // FIXME:+ fprintf(stderr, "node #%3d (%10.10s): %20.20s (%4.4s) [%4.4s %8.8s]:", i, ggml_op_name(node->op), node->name,+ fmt_size(ggml_nbytes(node)), node_allocr ? ggml_backend_name(node_backend) : "NULL", GET_CAUSE(node));+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ break;+ }+ ggml_tallocr_t src_allocr = node_allocr(src);+ ggml_backend_t src_backend = src_allocr ? get_allocr_backend(sched, src_allocr) : NULL;+ fprintf(stderr, " %20.20s (%4.4s) [%4.4s %8.8s]", src->name,+ fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));+ }+ fprintf(stderr, "\n");+ }+}++// creates a copy of the tensor with the same memory layout+static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {+ struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);+ for (int i = 0; i < GGML_MAX_DIMS; i++) {+ dup->nb[i] = tensor->nb[i];+ }+ return dup;+}++// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend+// TODO: merge passes+static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+ // reset state+ size_t hash_size = sched->hash_set.size;+ memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size);+ memset(sched->node_talloc, 0, sizeof(sched->node_talloc[0]) * hash_size);+ memset(sched->node_copies, 0, sizeof(sched->node_copies[0]) * hash_size);+ sched->n_splits = 0;++ struct ggml_init_params params = {+ /* .mem_size = */ sizeof(sched->context_buffer),+ /* .mem_buffer = */ sched->context_buffer,+ /* .no_alloc = */ true+ };++ if (sched->ctx != NULL) {+ ggml_free(sched->ctx);+ }++ sched->ctx = ggml_init(params);++ // pass 1: assign backends to ops with allocated inputs+ for (int i = 0; i < graph->n_leafs; i++) {+ struct ggml_tensor * leaf = graph->leafs[i];+ if (node_allocr(leaf) != NULL) {+ // do not overwrite user assignments+ continue;+ }+ ggml_backend_t leaf_backend = get_buffer_backend(sched, leaf->buffer);+ if (leaf_backend == NULL && leaf->view_src != NULL) {+ leaf_backend = get_buffer_backend(sched, leaf->view_src->buffer);+ }+ if (leaf_backend != NULL) {+ node_allocr(leaf) = ggml_backend_sched_get_tallocr(sched, leaf_backend);+ }+ }++ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ if (node_allocr(node) != NULL) {+ // do not overwrite user assignments+ continue;+ }+ ggml_backend_t node_backend = sched_backend_from_cur(sched, node);+ if (node_backend != NULL) {+ node_allocr(node) = ggml_backend_sched_get_tallocr(sched, node_backend);+ }+ }+ //printf("PASS 1 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);++ // pass 2: assign backends to ops from current assignments+ // TODO:+ // - reuse sched_backend_from_cur+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ ggml_tallocr_t node_allocr = node_allocr(node);+ if (node_allocr == NULL) {+ int cur_prio = INT_MAX;+ size_t cur_size = 0;+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ break;+ }+ ggml_tallocr_t src_allocr = node_allocr(src);+ if (src_allocr != NULL) {+ int src_prio = sched_allocr_prio(sched, src_allocr);+ size_t src_size = ggml_nbytes(src);+ if (src_prio < cur_prio && src_size >= cur_size) {+ cur_prio = src_prio;+ cur_size = src_size;+ node_allocr = src_allocr;+ SET_CAUSE(node, "2.src%d", j);+ }+ }+ }+ if (node_allocr != NULL) {+ node_allocr(node) = node_allocr;+ }+ }+ }+ //printf("PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);++ // pass 3: assign backends to remaining src from dst (should only be leafs)+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ ggml_tallocr_t node_allocr = node_allocr(node);+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ break;+ }+ ggml_tallocr_t src_allocr = node_allocr(src);+ if (src_allocr == NULL) {+ node_allocr(src) = node_allocr;+ }+ }+ }+ //printf("PASS 3 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);++ // pass 4: split graph, find tensors that need to be copied+ // TODO:+ // - when switching from a less preferred backend to a more preferred backend, check if it is possible to move the switch to an earlier point for the same cost+ // find first backend+ int cur_split = 0;+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ if (node->view_src == NULL) {+ sched->splits[0].tallocr = node_allocr(node);+ break;+ }+ }+ sched->splits[0].i_start = 0;+ sched->splits[0].n_inputs = 0;+ memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK+ ggml_tallocr_t cur_allocr = sched->splits[0].tallocr;+ size_t cur_backend_id = sched_allocr_prio(sched, cur_allocr);+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];++ if (ggml_is_view_op(node->op)) {+ continue;+ }++ ggml_tallocr_t node_allocr = node_allocr(node);++ if (node_allocr != cur_allocr) {+ sched->splits[cur_split].i_end = i;+ cur_split++;+ GGML_ASSERT(cur_split < GGML_MAX_SPLITS);+ sched->splits[cur_split].tallocr = node_allocr;+ sched->splits[cur_split].i_start = i;+ sched->splits[cur_split].n_inputs = 0;+ memset(sched->splits[cur_split].inputs, 0, sizeof(sched->splits[cur_split].inputs)); //HACK+ cur_allocr = node_allocr;+ cur_backend_id = sched_allocr_prio(sched, cur_allocr);+ }++ // find inputs that are not on the same backend+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ break;+ }+ ggml_tallocr_t src_allocr = node_allocr(src);+ if (src_allocr != node_allocr) {+ int n_inputs = sched->splits[cur_split].n_inputs++;+ GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS);+ sched->splits[cur_split].inputs[n_inputs] = (struct ggml_tensor *)src;++ // create copies+ size_t id = hash_id(src);+ if (sched->node_copies[id][cur_backend_id] == NULL) {+ struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);+ sched->node_copies[id][cur_backend_id] = tensor_copy;+ node_allocr(tensor_copy) = cur_allocr;+ ggml_backend_t backend = get_allocr_backend(sched, cur_allocr);+ ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name);+ }+ node->src[j] = sched->node_copies[id][cur_backend_id];+ }+ }+ }+ sched->splits[cur_split].i_end = graph->n_nodes;+ sched->n_splits = cur_split + 1;++ //fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); fflush(stdout);++#if 1+ // sanity check: all sources should have the same backend as the node+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ ggml_tallocr_t node_allocr = node_allocr(node);+ if (node_allocr == NULL) {+ fprintf(stderr, "!!!!!!! %s has no backend\n", node->name);+ }+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ break;+ }+ ggml_tallocr_t src_allocr = node_allocr(src);+ if (src_allocr != node_allocr /* && src_backend != NULL */) { // ignore nulls for now+ fprintf(stderr, "!!!! %s has backend %s, src %d (%s) has backend %s\n",+ node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL",+ j, src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL");+ }+ }+ }+#endif++ // create copies of the graph for each split+ // FIXME: avoid this copy, pass split inputs to ggml_gallocr_alloc_graph_n in some other way+ struct ggml_cgraph * graph_copy = ggml_new_graph_custom(sched->ctx, graph->n_nodes + sched->n_splits*GGML_MAX_SPLIT_INPUTS, false);+ for (int i = 0; i < sched->n_splits; i++) {+ struct ggml_backend_sched_split * split = &sched->splits[i];+ split->graph = ggml_graph_view(graph, split->i_start, split->i_end);++ // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split+ for (int j = 0; j < split->n_inputs; j++) {+ struct ggml_tensor * input = split->inputs[j];+ struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_allocr_prio(sched, split->tallocr)];+ input_cpy->src[0] = input;+ graph_copy->nodes[graph_copy->n_nodes++] = input_cpy;+ }++ for (int j = split->i_start; j < split->i_end; j++) {+ graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];+ }+ }+ sched->graph = graph_copy;+}++static void sched_alloc_splits(ggml_backend_sched_t sched) {+ ggml_gallocr_alloc_graph_n(+ sched->galloc,+ sched->graph,+ sched->hash_set,+ sched->node_talloc);+}++static void sched_compute_splits(ggml_backend_sched_t sched) {+ uint64_t copy_us[GGML_MAX_BACKENDS] = {0};+ uint64_t compute_us[GGML_MAX_BACKENDS] = {0};++ struct ggml_backend_sched_split * splits = sched->splits;++ for (int i = 0; i < sched->n_splits; i++) {+ struct ggml_backend_sched_split * split = &splits[i];+ ggml_backend_t split_backend = get_allocr_backend(sched, split->tallocr);+ int split_backend_id = sched_backend_prio(sched, split_backend);++ // copy the input tensors to the split backend+ uint64_t copy_start_us = ggml_time_us();+ for (int j = 0; j < split->n_inputs; j++) {+ struct ggml_tensor * input = split->inputs[j];+ struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_backend_prio(sched, split_backend)];+ if (input->buffer == NULL) {+ if (input->view_src == NULL) {+ fprintf(stderr, "input %s has no buffer and no view_src\n", input->name);+ exit(1);+ }+ // FIXME: may need to use the sched buffer instead+ ggml_backend_view_init(input->view_src->buffer, input);+ }+ if (input_cpy->buffer == NULL) {+ fprintf(stderr, "input_cpy %s has no buffer\n", input_cpy->name);+ exit(1);+ }+ //GGML_ASSERT(input->buffer->backend != input_cpy->buffer->backend);+ //GGML_ASSERT(input_cpy->buffer->backend == split_backend);+ ggml_backend_tensor_copy(input, input_cpy);+ }+ // ggml_backend_synchronize(split_backend);+ int64_t copy_end_us = ggml_time_us();+ copy_us[split_backend_id] += copy_end_us - copy_start_us;++#if 0+ char split_filename[GGML_MAX_NAME];+ snprintf(split_filename, GGML_MAX_NAME, "split_%i_%s.dot", i, ggml_backend_name(split_backend));+ ggml_graph_dump_dot(split->graph, NULL, split_filename);+#endif++ uint64_t compute_start_us = ggml_time_us();+ ggml_backend_graph_compute(split_backend, &split->graph);+ // ggml_backend_synchronize(split_backend);+ uint64_t compute_end_us = ggml_time_us();+ compute_us[split_backend_id] += compute_end_us - compute_start_us;+ }++#if 0+ // per-backend timings+ fprintf(stderr, "sched_compute_splits times (%d splits):\n", sched->n_splits);+ for (int i = 0; i < sched->n_backends; i++) {+ if (copy_us[i] > 0 || compute_us[i] > 0) {+ fprintf(stderr, "\t%5.5s: %lu us copy, %lu us compute\n", ggml_backend_name(sched->backends[i]), copy_us[i], compute_us[i]);+ }+ }+#endif+}++static void sched_reset(ggml_backend_sched_t sched) {+ for (int i = 0; i < sched->n_backends; i++) {+ ggml_tallocr_reset(sched->tallocs[i]);+ }+}++ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, int n_backends) {+ GGML_ASSERT(n_backends <= GGML_MAX_BACKENDS);++ struct ggml_backend_sched * sched = malloc(sizeof(struct ggml_backend_sched));+ memset(sched, 0, sizeof(struct ggml_backend_sched));++ sched->n_backends = n_backends;+ for (int i = 0; i < n_backends; i++) {+ sched->backends[i] = backends[i];+ }++ sched->galloc = ggml_gallocr_new();++ // init measure allocs for each backend+ for (int i = 0; i < n_backends; i++) {+ sched->tallocs[i] = ggml_tallocr_new_measure_from_backend(backends[i]);+ }++ return sched;+}++void ggml_backend_sched_free(ggml_backend_sched_t sched) {+ if (sched == NULL) {+ return;+ }+ for (int i = 0; i < sched->n_backends; i++) {+ ggml_tallocr_free(sched->tallocs[i]);+ }+ ggml_gallocr_free(sched->galloc);+ free(sched->hash_set.keys);+ free(sched->node_talloc);+ free(sched->node_copies);+ free(sched);+}++void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {+ // initialize hash tables+ size_t hash_size = measure_graph->visited_hash_table.size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS;+ sched->hash_set.size = hash_size;+ sched->hash_set.keys = malloc(sizeof(sched->hash_set.keys[0]) * hash_size);+ sched->node_talloc = malloc(sizeof(sched->node_talloc[0]) * hash_size);+ sched->node_copies = malloc(sizeof(sched->node_copies[0]) * hash_size);++ sched_split_graph(sched, measure_graph);+ sched_alloc_splits(sched);++ // allocate buffers and reset allocators+ for (int i = 0; i < sched->n_backends; i++) {+ size_t size = ggml_tallocr_max_size(sched->tallocs[i]);+ ggml_tallocr_free(sched->tallocs[i]);+ sched->tallocs[i] = ggml_tallocr_new_from_backend(sched->backends[i], size);+ }++ sched_reset(sched);+}++void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+ GGML_ASSERT(sched->hash_set.size >= graph->visited_hash_table.size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);++ sched_split_graph(sched, graph);+ sched_alloc_splits(sched);+ sched_compute_splits(sched);+ sched_reset(sched);+}++ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend) {+ int backend_index = sched_backend_prio(sched, backend);+ return sched->tallocs[backend_index];+}++ggml_backend_buffer_t ggml_backend_sched_get_buffer(ggml_backend_sched_t sched, ggml_backend_t backend) {+ int backend_index = sched_backend_prio(sched, backend);+ return ggml_tallocr_get_buffer(sched->tallocs[backend_index]);+}++void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {+ int backend_index = sched_backend_prio(sched, backend);+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);+ node_allocr(node) = sched->tallocs[backend_index];+}++// utils+void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {+ GGML_ASSERT(tensor->buffer == NULL);+ GGML_ASSERT(tensor->data == NULL);+ GGML_ASSERT(tensor->view_src != NULL);+ GGML_ASSERT(tensor->view_src->buffer != NULL);+ GGML_ASSERT(tensor->view_src->data != NULL);++ tensor->buffer = buffer;+ tensor->data = (char *)tensor->view_src->data + tensor->view_offs;+ tensor->backend = tensor->view_src->backend;+ ggml_backend_buffer_init_tensor(buffer, tensor);+}++void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) {+ GGML_ASSERT(tensor->buffer == NULL);+ GGML_ASSERT(tensor->data == NULL);+ GGML_ASSERT(tensor->view_src == NULL);+ GGML_ASSERT(addr >= ggml_backend_buffer_get_base(buffer));+ GGML_ASSERT((char *)addr + ggml_backend_buffer_get_alloc_size(buffer, tensor) <=+ (char *)ggml_backend_buffer_get_base(buffer) + ggml_backend_buffer_get_size(buffer));++ tensor->buffer = buffer;+ tensor->data = addr;+ ggml_backend_buffer_init_tensor(buffer, tensor);+}++static struct ggml_tensor * graph_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies,+ struct ggml_context * ctx_allocated, struct ggml_context * ctx_unallocated, struct ggml_tensor * src) {++ GGML_ASSERT(src != NULL);+ GGML_ASSERT(src->data && "graph must be allocated");++ size_t id = ggml_hash_insert(hash_set, src);+ if (id == GGML_HASHTABLE_ALREADY_EXISTS) {+ return node_copies[ggml_hash_find(hash_set, src)];+ }++ struct ggml_tensor * dst = ggml_dup_tensor_layout(src->data && !src->view_src ? ctx_allocated : ctx_unallocated, src);+ if (src->view_src != NULL) {+ dst->view_src = graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src);+ dst->view_offs = src->view_offs;+ }+ dst->op = src->op;+ memcpy(dst->op_params, src->op_params, sizeof(dst->op_params));+ ggml_set_name(dst, src->name);++ // copy src+ for (int i = 0; i < GGML_MAX_SRC; i++) {+ struct ggml_tensor * s = src->src[i];+ if (s == NULL) {+ break;+ }+ dst->src[i] = graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s);+ }++ node_copies[id] = dst;+ return dst;+}++static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) {+ size_t id = ggml_hash_find(hash_set, src);+ if (node_init[id]) {+ return;+ }+ node_init[id] = true;++ struct ggml_tensor * dst = node_copies[id];+ if (dst->view_src != NULL) {+ ggml_backend_view_init(dst->view_src->buffer, dst);+ }+ else {+ ggml_backend_tensor_copy(src, dst);+ }++ // init src+ for (int i = 0; i < GGML_MAX_SRC; i++) {+ struct ggml_tensor * s = src->src[i];+ if (s == NULL) {+ break;+ }+ graph_init_tensor(hash_set, node_copies, node_init, s);+ }+}++struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph) {+ struct ggml_hash_set hash_set = {+ /* .size = */ graph->visited_hash_table.size,+ /* .keys = */ calloc(sizeof(hash_set.keys[0]) * graph->visited_hash_table.size, 1)+ };+ struct ggml_tensor ** node_copies = calloc(sizeof(node_copies[0]) * hash_set.size, 1);+ bool * node_init = calloc(sizeof(node_init[0]) * hash_set.size, 1);++ struct ggml_init_params params = {+ /* .mem_size = */ ggml_tensor_overhead()*hash_set.size + ggml_graph_overhead_custom(graph->size, false),+ /* .mem_buffer = */ NULL,+ /* .no_alloc = */ true+ };++ struct ggml_context * ctx_allocated = ggml_init(params);+ struct ggml_context * ctx_unallocated = ggml_init(params);++ // dup nodes+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node);+ }++ // allocate nodes+ ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_allocated, backend);++ //printf("copy buffer size: %zu MB\n", ggml_backend_buffer_get_size(buffer) / 1024 / 1024);++ // copy data and init views+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ graph_init_tensor(hash_set, node_copies, node_init, node);+ }++ // build graph copy+ struct ggml_cgraph * graph_copy = ggml_new_graph_custom(ctx_allocated, graph->size, false);+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ struct ggml_tensor * node_copy = node_copies[ggml_hash_find(hash_set, node)];+ graph_copy->nodes[i] = node_copy;+ }+ graph_copy->n_nodes = graph->n_nodes;++ free(hash_set.keys);+ free(node_copies);+ free(node_init);++ return (struct ggml_backend_graph_copy) {+ /* .buffer = */ buffer,+ /* .ctx_allocated = */ ctx_allocated,+ /* .ctx_unallocated = */ ctx_unallocated,+ /* .graph = */ graph_copy,+ };+}++void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy) {+ ggml_backend_buffer_free(copy.buffer);+ ggml_free(copy.ctx_allocated);+ ggml_free(copy.ctx_unallocated);+}++void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data) {+ struct ggml_backend_graph_copy copy = ggml_backend_graph_copy(backend2, graph);+ struct ggml_cgraph * g1 = graph;+ struct ggml_cgraph * g2 = copy.graph;++ assert(g1->n_nodes == g2->n_nodes);++ for (int i = 0; i < g1->n_nodes; i++) {+ //printf("eval %d/%d\n", i, g1->n_nodes);+ struct ggml_tensor * t1 = g1->nodes[i];+ struct ggml_tensor * t2 = g2->nodes[i];++ assert(t1->op == t2->op && ggml_are_same_layout(t1, t2));++ struct ggml_cgraph g1v = ggml_graph_view(g1, i, i + 1);+ struct ggml_cgraph g2v = ggml_graph_view(g2, i, i + 1);++ ggml_backend_graph_compute(backend1, &g1v);+ ggml_backend_graph_compute(backend2, &g2v);++ if (ggml_is_view_op(t1->op)) {+ continue;+ }++ // compare results, calculate rms etc+ if (!callback(i, t1, t2, user_data)) {+ break;+ }+ }++ ggml_backend_graph_copy_free(copy);}diff --git a/llama.cpp/ggml-backend.h b/llama.cpp/ggml-backend.hindex da134b0d..58d5ccae 100644--- a/llama.cpp/ggml-backend.h+++ b/llama.cpp/ggml-backend.h@@ -1,115 +1,50 @@#pragma once#include "ggml.h"+#include "ggml-alloc.h"#ifdef __cplusplusextern "C" {#endif- struct ggml_backend;- struct ggml_backend_buffer;- // type-erased backend-specific types / wrappers- typedef void * ggml_backend_context_t;- typedef void * ggml_backend_graph_plan_t;- typedef void * ggml_backend_buffer_context_t;-- // avoid accessing internals of these types- typedef struct ggml_backend * ggml_backend_t;+ typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;typedef struct ggml_backend_buffer * ggml_backend_buffer_t;+ typedef struct ggml_backend * ggml_backend_t;+ typedef void * ggml_backend_graph_plan_t;//- // backend buffer+ // Backend buffer//- struct ggml_backend_buffer_i {- void (*free_buffer) (ggml_backend_buffer_t buffer);- void * (*get_base) (ggml_backend_buffer_t buffer); // get base pointer- size_t (*get_alloc_size)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); // pre-allocation callback- void (*init_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); // post-allocation callback- void (*free_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); // pre-free callback- };-- // TODO: hide behind API- struct ggml_backend_buffer {- struct ggml_backend_buffer_i iface;-- ggml_backend_t backend;- ggml_backend_buffer_context_t context;-- size_t size;- };-- // backend buffer functions- GGML_API ggml_backend_buffer_t ggml_backend_buffer_init(- struct ggml_backend * backend,- struct ggml_backend_buffer_i iface,- ggml_backend_buffer_context_t context,- size_t size);+ // buffer type+ GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);+ GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);+ GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);+ GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);+ // bufferGGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);- GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);- GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);- GGML_API void ggml_backend_buffer_free_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);+ GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer);//- // backend+ // Backend//- struct ggml_backend_i {- const char * (*get_name)(ggml_backend_t backend);-- void (*free)(ggml_backend_t backend);-- // buffer allocation- ggml_backend_buffer_t (*alloc_buffer)(ggml_backend_t backend, size_t size);-- // get buffer alignment- size_t (*get_alignment)(ggml_backend_t backend);-- // tensor data access- // these functions can be asynchronous, helper functions are provided for synchronous access that automatically call synchronize- void (*set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);- void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);- void (*synchronize) (ggml_backend_t backend);-- // (optional) copy tensor between different backends, allow for single-copy tranfers- void (*cpy_tensor_from)(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);- void (*cpy_tensor_to) (ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);-- // compute graph with a plan- ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, struct ggml_cgraph * cgraph);- void (*graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);- void (*graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);-- // compute graph without a plan- void (*graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph);-- // check if the backend supports an operation- bool (*supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);- };-- // TODO: hide behind API- struct ggml_backend {- struct ggml_backend_i iface;-- ggml_backend_context_t context;- };-- // backend helper functions- GGML_API ggml_backend_t ggml_get_backend(const struct ggml_tensor * tensor);GGML_API const char * ggml_backend_name(ggml_backend_t backend);GGML_API void ggml_backend_free(ggml_backend_t backend);- GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);+ GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);+ GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);+ GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);- GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);-- GGML_API void ggml_backend_tensor_set_async( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);- GGML_API void ggml_backend_tensor_get_async(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);@@ -125,6 +60,7 @@ extern "C" {// tensor copy between different backendsGGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);+ GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst); // automatic fallback to sync copy//// CPU backend@@ -133,10 +69,112 @@ extern "C" {GGML_API ggml_backend_t ggml_backend_cpu_init(void);GGML_API bool ggml_backend_is_cpu(ggml_backend_t backend);-GGML_API void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);- GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(ggml_backend_t backend_cpu, void * ptr, size_t size);+ // Create a backend buffer from an existing pointer+ GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);++ GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);++ //+ // Backend registry+ //++ // The backend registry is a registry of all the available backends, and allows initializing backends in a generic way++ GGML_API size_t ggml_backend_reg_get_count(void);+ GGML_API size_t ggml_backend_reg_find_by_name(const char * name);+ GGML_API ggml_backend_t ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is name[:params]+ GGML_API const char * ggml_backend_reg_get_name(size_t i);+ GGML_API ggml_backend_t ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific+ GGML_API ggml_backend_buffer_type_t ggml_backend_reg_get_default_buffer_type(size_t i);+ GGML_API ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size);++ //+ // Backend scheduler+ //++ // The backend scheduler allows for multiple backends to be used together+ // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends+ // The backends are selected based on:+ // - the backend that supports the operation+ // - the location of the pre-allocated tensors (e.g. the weights)+ /*+ Example usage:++ sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, num_backends);+ // sched is initialized with measure allocators and cannot be used until allocated with a measure graph++ // initialize buffers from a measure graph+ measure_graph = build_graph(sched); // use the allocr to allocate inputs as needed++ // in build_graph:+ build_graph(...) {+ // allocating tensors in a specific backend (optional, recommended: pre-allocate inputs in a different buffer)+ alloc_cpu = ggml_backend_sched_get_allocr(sched, backend_cpu);+ ggml_allocr_alloc(alloc_cpu, tensor);++ // manually assigning nodes to a backend (optional, shouldn't be needed in most cases)+ struct ggml_tensor * node = ggml_mul_mat(ctx, ...);+ ggml_backend_sched_set_node_backend(sched, node, backend_gpu);+ }++ // allocate backend buffers from measure graph+ ggml_backend_sched_init_measure(sched, measure_graph);++ // the scheduler is now ready to compute graphs++ // compute+ graph = build_graph(sched);+ ggml_backend_sched_graph_compute(sched, graph);+ */++ struct ggml_backend_sched;+ typedef struct ggml_backend_sched * ggml_backend_sched_t;++ // Initialize a backend scheduler+ GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, int n_backends);++ GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);++ // Initialize backend buffers from a measure graph+ GGML_API void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);++ GGML_API ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend);+ GGML_API ggml_backend_buffer_t ggml_backend_sched_get_buffer (ggml_backend_sched_t sched, ggml_backend_t backend);++ GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);++ // Allocate a graph on the backend scheduler+ GGML_API void ggml_backend_sched_graph_compute(+ ggml_backend_sched_t sched,+ struct ggml_cgraph * graph);+++ //+ // Utils+ //++ struct ggml_backend_graph_copy {+ ggml_backend_buffer_t buffer;+ struct ggml_context * ctx_allocated;+ struct ggml_context * ctx_unallocated;+ struct ggml_cgraph * graph;+ };++ // Copy a graph to a different backend+ GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);+ GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);++ typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);++ // Compare the output of two backends+ GGML_API void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);++ // Tensor initialization+ GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);+ GGML_API void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+#ifdef __cplusplus}diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 40497c49..f31917ab 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -1,16 +1,23 @@// -*- mode:cuda;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+// vi: set et ft=cuda ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include <algorithm>+#include <assert.h>+#include <atomic>+#include <cinttypes>#include <cstddef>#include <cstdint>+#include <float.h>#include <limits>#include <stdint.h>#include <stdio.h>-#include <atomic>-#include <assert.h>+#include <vector>-#if defined(GGML_USE_HIPBLAS)+#ifdef __i386__+#error "you need to use a 64-bit compiler for llamafile"+#endif+#if defined(GGML_USE_HIPBLAS)#include <hip/hip_runtime.h>#include <hipblas/hipblas.h>#include <hip/hip_fp16.h>@@ -71,6 +78,7 @@#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize#define cudaSetDevice hipSetDevice#define cudaStreamCreateWithFlags hipStreamCreateWithFlags+#define cudaStreamFireAndForget hipStreamFireAndForget#define cudaStreamNonBlocking hipStreamNonBlocking#define cudaStreamSynchronize hipStreamSynchronize#define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags)@@ -86,14 +94,12 @@#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx#else-#include <cuda_runtime.h>#include <cublas_v2.h>#include <cuda_fp16.h>+#endif // defined(GGML_USE_HIPBLAS)-#endif // defined(GGML_USE_HIPBLAS) || defined(GGML_USE_TINYBLAS)--#if defined(GGML_USE_TINYBLAS)+#ifdef GGML_USE_TINYBLAS#define CUBLAS_ENTRY() cudaStream_t _ggml_stream = nullptr#define CUBLAS_SET_STREAM(_, stream) do _ggml_stream = (stream); while (0)#define CUBLAS_HANDLE(_) _ggml_stream@@ -105,72 +111,16 @@#include "ggml-cuda.h"#include "ggml.h"--#ifdef __i386__-#error "you need to use a 64-bit compiler for llamafile"-#endif--////////////////////////////////////////////////////////////////////////////////-// BEGIN: COPIED FROM GGML.C--#define QK4_0 32-#define QK4_1 32-#define QK5_0 32-#define QK5_1 32-#define QK8_0 32-#define QK8_1 32-#define QK_K 256+#include "ggml-backend-impl.h"#undef MIN#undef MAX+#define MIN(a, b) ((a) < (b) ? (a) : (b))#define MAX(a, b) ((a) > (b) ? (a) : (b))-#ifndef M_PI-#define M_PI 3.14159265358979323846-#endif--#define ggml_blck_size ggml_blck_size_-static int ggml_blck_size(enum ggml_type type) {- switch (type) {- case GGML_TYPE_I8:- return 1;- case GGML_TYPE_I16:- return 1;- case GGML_TYPE_I32:- return 1;- case GGML_TYPE_F32:- return 1;- case GGML_TYPE_F16:- return 1;- case GGML_TYPE_Q4_0:- return QK4_0;- case GGML_TYPE_Q4_1:- return QK4_1;- case GGML_TYPE_Q5_0:- return QK5_0;- case GGML_TYPE_Q5_1:- return QK5_1;- case GGML_TYPE_Q8_0:- return QK8_0;- case GGML_TYPE_Q8_1:- return QK8_1;- case GGML_TYPE_Q2_K:- return QK_K;- case GGML_TYPE_Q3_K:- return QK_K;- case GGML_TYPE_Q4_K:- return QK_K;- case GGML_TYPE_Q5_K:- return QK_K;- case GGML_TYPE_Q6_K:- return QK_K;- case GGML_TYPE_Q8_K:- return QK_K;- default:- return 0;- }-}+////////////////////////////////////////////////////////////////////////////////+// BEGIN: COPIED FROM GGML.C#define ggml_is_quantized ggml_is_quantized_static bool ggml_is_quantized(enum ggml_type type) {@@ -215,45 +165,14 @@ static bool ggml_is_quantized(enum ggml_type type) {}#define ggml_type_size ggml_type_size_-static size_t ggml_type_size(enum ggml_type type) {- switch (type) {- case GGML_TYPE_I8:- return sizeof(int8_t);- case GGML_TYPE_I16:- return sizeof(int16_t);- case GGML_TYPE_I32:- return sizeof(int32_t);- case GGML_TYPE_F32:- return sizeof(float);- case GGML_TYPE_F16:- return 2;- case GGML_TYPE_Q4_0:- return 18;- case GGML_TYPE_Q4_1:- return 20;- case GGML_TYPE_Q5_0:- return 22;- case GGML_TYPE_Q5_1:- return 24;- case GGML_TYPE_Q8_0:- return 34;- case GGML_TYPE_Q8_1:- return 40;- case GGML_TYPE_Q2_K:- return 84;- case GGML_TYPE_Q3_K:- return 110;- case GGML_TYPE_Q4_K:- return 144;- case GGML_TYPE_Q5_K:- return 176;- case GGML_TYPE_Q6_K:- return 210;- case GGML_TYPE_Q8_K:- return 292;- default:- return 0;- }+static size_t ggml_type_size(enum ggml_type type);++#define ggml_blck_size ggml_blck_size_+static int ggml_blck_size(enum ggml_type type);++#define ggml_is_transposed ggml_is_transposed_+static bool ggml_is_transposed(const struct ggml_tensor * tensor) {+ return tensor->nb[0] > tensor->nb[1];}#define ggml_nbytes ggml_nbytes_@@ -275,24 +194,23 @@ static size_t ggml_nbytes(const struct ggml_tensor * tensor) {return nbytes;}-#define ggml_is_transposed ggml_is_transposed_-static bool ggml_is_transposed(const struct ggml_tensor * tensor) {- return tensor->nb[0] > tensor->nb[1];+static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {+ return ((const int32_t *)(tensor->op_params))[i];+}++#define ggml_get_unary_op ggml_get_unary_op_+static enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) {+ return (enum ggml_unary_op) ggml_get_op_params_i32(tensor, 0);}-#define ggml_is_permuted ggml_is_permuted_-static bool ggml_is_permuted(const struct ggml_tensor * tensor) {- return tensor->nb[0] > tensor->nb[1] || tensor->nb[1] > tensor->nb[2] || tensor->nb[2] > tensor->nb[3];+#define ggml_nbytes_split ggml_nbytes_split_+static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {+ return (nrows_split*tensor->ne[0]*ggml_type_size(tensor->type))/ggml_blck_size(tensor->type);}-#define ggml_is_contiguous ggml_is_contiguous_-static bool ggml_is_contiguous(const struct ggml_tensor * tensor) {- static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");- return- tensor->nb[0] == ggml_type_size(tensor->type) &&- tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&- tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&- tensor->nb[3] == tensor->nb[2]*tensor->ne[2];+#define ggml_nelements ggml_nelements_+static int64_t ggml_nelements(const struct ggml_tensor * tensor) {+ return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3];}#define ggml_nrows ggml_nrows_@@ -300,23 +218,28 @@ static int64_t ggml_nrows(const struct ggml_tensor * tensor) {return tensor->ne[1]*tensor->ne[2]*tensor->ne[3];}-#define ggml_nelements ggml_nelements_-static int64_t ggml_nelements(const struct ggml_tensor * tensor) {- return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3];+#define ggml_is_contiguous ggml_is_contiguous_+static bool ggml_is_contiguous(const struct ggml_tensor * tensor) {+ return+ tensor->nb[0] == ggml_type_size(tensor->type) &&+ tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&+ tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&+ tensor->nb[3] == tensor->nb[2]*tensor->ne[2];}-static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {- return ((const int32_t *)(tensor->op_params))[i];+#define ggml_op_name ggml_op_name_+static const char * ggml_op_name(enum ggml_op op) {+ return "REDACTED!GGML_OP_NAME[op]";}-#define ggml_get_unary_op ggml_get_unary_op_-static enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) {- return (enum ggml_unary_op) ggml_get_op_params_i32(tensor, 0);+#define ggml_type_name ggml_type_name_+static const char * ggml_type_name(enum ggml_type type) {+ return "REDACTED!GGML_TYPE_NAME[type]";}-#define ggml_nbytes_split ggml_nbytes_split_-static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {- return (nrows_split*tensor->ne[0]*ggml_type_size(tensor->type))/ggml_blck_size(tensor->type);+#define ggml_element_size ggml_element_size_+static size_t ggml_element_size(const struct ggml_tensor * tensor) {+ return ggml_type_size(tensor->type);}static float ggml_rope_yarn_corr_dim(int n_dims, int n_orig_ctx, float n_rot, float base) {@@ -335,11 +258,162 @@ static void ggml_rope_yarn_corr_dims(// END: COPIED FROM GGML.C////////////////////////////////////////////////////////////////////////////////+////////////////////////////////////////////////////////////////////////////////+// BEGIN: COPIED FROM GGML-BACKEND.C++#define ggml_backend_buffer_init ggml_backend_buffer_init_+static ggml_backend_buffer_t ggml_backend_buffer_init(+ ggml_backend_buffer_type_t buft,+ struct ggml_backend_buffer_i iface,+ ggml_backend_buffer_context_t context,+ size_t size) {+ fprintf(stderr, "WARNING: using untested foreign malloc due to ggml backend\n");+ ggml_backend_buffer_t buffer = (ggml_backend_buffer_t)malloc(sizeof(struct ggml_backend_buffer));+ (*buffer) = (struct ggml_backend_buffer) {+ /* .interface = */ iface,+ /* .buft = */ buft,+ /* .context = */ context,+ /* .size = */ size,+ };+ return buffer;+}++#define ggml_backend_cpu_buffer_type ggml_backend_cpu_buffer_type_+static ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);++static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer);+static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+static void ggml_backend_cpu_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);+static void ggml_backend_cpu_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);++// for buffers from ptr, free is not called+static struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = {+ /* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,+ /* .init_tensor = */ NULL, // no initialization required+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,+ /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,+ /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,+};++#define ggml_backend_cpu_buffer_from_ptr ggml_backend_cpu_buffer_from_ptr_+static ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {+ return ggml_backend_buffer_init(ggml_backend_cpu_buffer_type(), cpu_backend_buffer_i_from_ptr, ptr, size);+}++static const char * ggml_backend_cpu_name(ggml_backend_t backend) {+ return "CPU";+ GGML_UNUSED(backend);+}++#define ggml_backend_is_cpu ggml_backend_is_cpu_+static bool ggml_backend_is_cpu(ggml_backend_t backend) {+ return backend->iface.get_name == ggml_backend_cpu_name;+}++static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512++static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+ return TENSOR_ALIGNMENT;+ GGML_UNUSED(buft);+}++static bool ggml_backend_cpu_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ return ggml_backend_is_cpu(backend);+ GGML_UNUSED(buft);+}++static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ fprintf(stderr, "WARNING: using untested foreign free due to ggml backend\n");+ free(buffer->context);+ GGML_UNUSED(buffer);+}++static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {+ return (void *)buffer->context;+}++static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ memcpy((char *)tensor->data + offset, data, size);+ GGML_UNUSED(buffer);+}++static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ memcpy(data, (const char *)tensor->data + offset, size);+ GGML_UNUSED(buffer);+}++#define ggml_backend_tensor_get ggml_backend_tensor_get_+static void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ tensor->buffer->iface.get_tensor(tensor->buffer, tensor, data, offset, size);+}++static void ggml_backend_cpu_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {+ ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));+ GGML_UNUSED(buffer);+}++#define ggml_backend_tensor_set ggml_backend_tensor_set_+static void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ tensor->buffer->iface.set_tensor(tensor->buffer, tensor, data, offset, size);+}++static void ggml_backend_cpu_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {+ ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));+ GGML_UNUSED(buffer);+}++static struct ggml_backend_buffer_i cpu_backend_buffer_i = {+ /* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,+ /* .init_tensor = */ NULL, // no initialization required+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,+ /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,+ /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,+};++static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ size += TENSOR_ALIGNMENT; // malloc may return an address that is not aligned+ fprintf(stderr, "WARNING: using untested foreign malloc due to ggml backend\n");+ void * data = malloc(size); // TODO: maybe use GGML_ALIGNED_MALLOC?+ return ggml_backend_buffer_init(buft, cpu_backend_buffer_i, data, size);+}++static ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {+ static struct ggml_backend_buffer_type ggml_backend_buffer_type_cpu = {+ /* .iface = */ {+ /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes+ /* .supports_backend = */ ggml_backend_cpu_buffer_type_supports_backend,+ },+ /* .context = */ NULL,+ };+ return &ggml_backend_buffer_type_cpu;+}++#define ggml_backend_buft_get_alloc_size ggml_backend_buft_get_alloc_size_+static size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) {+ // get_alloc_size is optional, defaults to ggml_nbytes+ if (buft->iface.get_alloc_size) {+ return buft->iface.get_alloc_size(buft, tensor);+ }+ return ggml_nbytes(tensor);+}++// END: COPIED FROM GGML-BACKEND.C+////////////////////////////////////////////////////////////////////////////////+#define MIN_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products#define CC_VOLTA 700#define CC_OFFSET_AMD 1000000#define CC_RDNA2 (CC_OFFSET_AMD + 1030)+#define GGML_CUDA_MAX_NODES 8192+// define this if you want to always fallback to MMQ kernels and not use cuBLAS for matrix multiplication// on modern hardware, using cuBLAS is recommended as it utilizes F16 tensor cores which are very performant// for large computational tasks. the drawback is that this requires some extra amount of VRAM:@@ -439,11 +513,11 @@ static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");fprintf(stderr, "\nCUDA error %d at %s:%d: %s\n", err_, __FILE__, __LINE__, \cudaGetErrorString(err_)); \fprintf(stderr, "current device: %d\n", id); \- exit(1); \+ GGML_ASSERT(!"CUDA error"); \} \} while (0)-#if CUDART_VERSION >= 12000 && !defined(GGML_USE_TINYBLAS)+#if CUDART_VERSION >= 12000#define CUBLAS_CHECK(err) \do { \cublasStatus_t err_ = (err); \@@ -453,7 +527,7 @@ static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");fprintf(stderr, "\ncuBLAS error %d at %s:%d: %s\n", \err_, __FILE__, __LINE__, cublasGetStatusString(err_)); \fprintf(stderr, "current device: %d\n", id); \- exit(1); \+ GGML_ASSERT(!"cuBLAS error"); \} \} while (0)#else@@ -465,10 +539,10 @@ static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");cudaGetDevice(&id); \fprintf(stderr, "\ncuBLAS error %d at %s:%d\n", err_, __FILE__, __LINE__); \fprintf(stderr, "current device: %d\n", id); \- exit(1); \+ GGML_ASSERT(!"cuBLAS error"); \} \} while (0)-#endif // CUDART_VERSION >= 11 && !defined(GGML_USE_TINYBLAS)+#endif // CUDART_VERSION >= 11#if CUDART_VERSION >= 11100#define GGML_CUDA_ASSUME(x) __builtin_assume(x)@@ -485,7 +559,7 @@ typedef float2 dfloat2;#endif //GGML_CUDA_F16static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {- const uint16_t * x16 = (uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignmentint x32 = 0;x32 |= x16[0] << 0;@@ -495,7 +569,7 @@ static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const}static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {- const uint16_t * x16 = (uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignmentint x32 = 0;x32 |= x16[0] << 0;@@ -505,11 +579,11 @@ static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, con}static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {- return *((int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment}static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {- return *((int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment}template<typename T>@@ -682,14 +756,15 @@ static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_#define WARP_SIZE 32#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses-#define CUDA_ADD_BLOCK_SIZE 256-#define CUDA_MUL_BLOCK_SIZE 256#define CUDA_GELU_BLOCK_SIZE 256#define CUDA_SILU_BLOCK_SIZE 256+#define CUDA_RELU_BLOCK_SIZE 256+#define CUDA_SQR_BLOCK_SIZE 256#define CUDA_CPY_BLOCK_SIZE 32#define CUDA_SCALE_BLOCK_SIZE 256#define CUDA_CLAMP_BLOCK_SIZE 256#define CUDA_ROPE_BLOCK_SIZE 256+#define CUDA_SOFT_MAX_BLOCK_SIZE 1024#define CUDA_ALIBI_BLOCK_SIZE 32#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32#define CUDA_QUANTIZE_BLOCK_SIZE 256@@ -717,7 +792,7 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA#define MUL_MAT_SRC1_COL_STRIDE 128#define MAX_STREAMS 8-static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { nullptr };+static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { { nullptr } };struct ggml_tensor_extra_gpu {void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors@@ -746,44 +821,116 @@ static void * g_scratch_buffer = nullptr;static size_t g_scratch_size = 0; // disabled by defaultstatic size_t g_scratch_offset = 0;-#if !defined(GGML_USE_TINYBLAS)+#ifndef GGML_USE_TINYBLASstatic cublasHandle_t g_cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};-#endif // !defined(GGML_USE_TINYBLAS)--static __global__ void add_f32(const float * x, const float * y, float * dst, const int kx, const int ky) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+#endif- if (i >= kx) {- return;+static __device__ __forceinline__ float warp_reduce_sum(float x) {+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ x += __shfl_xor_sync(0xffffffff, x, mask, 32);}- dst[i] = x[i] + y[i%ky];+ return x;}-static __global__ void add_f16_f32_f16(const half * x, const float * y, half * dst, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ a.x += __shfl_xor_sync(0xffffffff, a.x, mask, 32);+ a.y += __shfl_xor_sync(0xffffffff, a.y, mask, 32);+ }+ return a;+}- if (i >= k) {- return;+static __device__ __forceinline__ float warp_reduce_max(float x) {+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, mask, 32));}- dst[i] = __hadd(x[i], __float2half(y[i]));+ return x;}-static __global__ void add_f16_f32_f32(const half * x, const float * y, float * dst, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+static __device__ __forceinline__ float op_repeat(const float a, const float b) {+ return b;+}- if (i >= k) {+static __device__ __forceinline__ float op_add(const float a, const float b) {+ return a + b;+}++static __device__ __forceinline__ float op_mul(const float a, const float b) {+ return a * b;+}++static __device__ __forceinline__ float op_div(const float a, const float b) {+ return a / b;+}++template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>+static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,+ int ne0, int ne1, int ne2, int ne3,+ int ne10, int ne11, int ne12, int ne13,+ /*int s0, */ int s1, int s2, int s3,+ /*int s10,*/ int s11, int s12, int s13) {+ const int i0s = blockDim.x*blockIdx.x + threadIdx.x;+ const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);+ const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;+ const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;++ if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {return;}- dst[i] = __half2float(x[i]) + y[i];++ const int i11 = i1 % ne11;+ const int i12 = i2 % ne12;+ const int i13 = i3 % ne13;++ const size_t i_src0 = i3*s3 + i2*s2 + i1*s1;+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;+ const size_t i_dst = i_src0;++ const src0_t * src0_row = src0 + i_src0;+ const src1_t * src1_row = src1 + i_src1;+ dst_t * dst_row = dst + i_dst;++ for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {+ const int i10 = i0 % ne10;+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);+ }}-static __global__ void mul_f32(const float * x, const float * y, float * dst, const int kx, const int ky) {+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>+static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,+ int ne0, int ne1, int ne2, int ne3,+ int ne10, int ne11, int ne12, int ne13,+ /*int s0, */ int s1, int s2, int s3,+ /*int s10,*/ int s11, int s12, int s13) {+const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= kx) {+ const int i3 = i/(ne2*ne1*ne0);+ const int i2 = (i/(ne1*ne0)) % ne2;+ const int i1 = (i/ne0) % ne1;+ const int i0 = i % ne0;++ if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {return;}- dst[i] = x[i] * y[i%ky];++ const int i11 = i1 % ne11;+ const int i12 = i2 % ne12;+ const int i13 = i3 % ne13;++ const size_t i_src0 = i3*s3 + i2*s2 + i1*s1;+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;+ const size_t i_dst = i_src0;++ const src0_t * src0_row = src0 + i_src0;+ const src1_t * src1_row = src1 + i_src1;+ dst_t * dst_row = dst + i_dst;++ const int i10 = i0 % ne10;+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);}static __global__ void gelu_f32(const float * x, float * dst, const int k) {@@ -808,22 +955,29 @@ static __global__ void silu_f32(const float * x, float * dst, const int k) {dst[i] = x[i] / (1.0f + expf(-x[i]));}-static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- a.x += __shfl_xor_sync(0xffffffff, a.x, mask, 32);- a.y += __shfl_xor_sync(0xffffffff, a.y, mask, 32);+static __global__ void relu_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;}- return a;+ dst[i] = fmaxf(x[i], 0);+}++static __global__ void sqr_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;+ }+ dst[i] = x[i] * x[i];}template <int block_size>-static __global__ void norm_f32(const float * x, float * dst, const int ncols) {+static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {const int row = blockIdx.x*blockDim.y + threadIdx.y;const int tid = threadIdx.x;- const float eps = 1e-5f;-float2 mean_var = make_float2(0.f, 0.f);for (int col = tid; col < ncols; col += block_size) {@@ -855,14 +1009,6 @@ static __global__ void norm_f32(const float * x, float * dst, const int ncols) {}}-static __device__ __forceinline__ float warp_reduce_sum(float x) {-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- x += __shfl_xor_sync(0xffffffff, x, mask, 32);- }- return x;-}-template <int block_size>static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {const int row = blockIdx.x*blockDim.y + threadIdx.y;@@ -1861,31 +2007,65 @@ static __global__ void quantize_q8_1(const float * __restrict__ x, void * __rest}template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static __global__ void k_get_rows(const void * x, const int32_t * y, dst_t * dst, const int ncols) {- const int col = (blockIdx.x*blockDim.x + threadIdx.x)*2;- const int row = blockDim.y*blockIdx.y + threadIdx.y;-- if (col >= ncols) {+static __global__ void k_get_rows(+ const void * src0, const int32_t * src1, dst_t * dst,+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {++ const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;++ if (i00 >= ne00) {return;}- const int r = y[row];+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];- // copy x[r*ncols + col] to dst[row*ncols + col]- const int xi = r*ncols + col;- const int di = row*ncols + col;+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;- const int ib = xi/qk; // block index- const int iqs = (xi%qk)/qr; // quant index- const int iybs = di - di%qk; // y block start index+ const int ib = i00/qk; // block index+ const int iqs = (i00%qk)/qr; // quant index+ const int iybs = i00 - i00%qk; // dst block start indexconst int y_offset = qr == 1 ? 1 : qk/2;// dequantizedfloat2 v;- dequantize_kernel(x, ib, iqs, v);+ dequantize_kernel(src0_row, ib, iqs, v);++ dst_row[iybs + iqs + 0] = v.x;+ dst_row[iybs + iqs + y_offset] = v.y;+}++template<typename src0_t, typename dst_t>+static __global__ void k_get_rows_float(+ const src0_t * src0, const int32_t * src1, dst_t * dst,+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {- dst[iybs + iqs + 0] = v.x;- dst[iybs + iqs + y_offset] = v.y;+ const int i00 = blockIdx.x*blockDim.x + threadIdx.x;+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;++ if (i00 >= ne00) {+ return;+ }++ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];++ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);++ dst_row[i00] = src0_row[i00];}template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>@@ -2480,6 +2660,7 @@ static __device__ __forceinline__ float vec_dot_q4_0_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh; (void)x_sc;__shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];__shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0];@@ -2491,7 +2672,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {-+ (void)x_qh; (void)x_sc;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);GGML_CUDA_ASSUME(k >= 0);@@ -2500,7 +2681,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI4_0;const int kqsx = k % QI4_0;- const block_q4_0 * bx0 = (block_q4_0 *) vx;+ const block_q4_0 * bx0 = (const block_q4_0 *) vx;float * x_dmf = (float *) x_dm;@@ -2538,9 +2719,10 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh; (void)x_sc;const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));- const float * x_dmf = (float *) x_dm;+ const float * x_dmf = (const float *) x_dm;int u[2*VDR_Q4_0_Q8_1_MMQ];@@ -2574,6 +2756,7 @@ static __device__ __forceinline__ float vec_dot_q4_1_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh; (void)x_sc;__shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y];__shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1];@@ -2585,6 +2768,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh; (void)x_sc;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -2594,7 +2778,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI4_1;const int kqsx = k % QI4_1;- const block_q4_1 * bx0 = (block_q4_1 *) vx;+ const block_q4_1 * bx0 = (const block_q4_1 *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -2629,6 +2813,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh; (void)x_sc;const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));@@ -2666,6 +2851,7 @@ static __device__ __forceinline__ float vec_dot_q5_0_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh; (void)x_sc;__shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];__shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0];@@ -2677,6 +2863,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh; (void)x_sc;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -2686,7 +2873,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI5_0;const int kqsx = k % QI5_0;- const block_q5_0 * bx0 = (block_q5_0 *) vx;+ const block_q5_0 * bx0 = (const block_q5_0 *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -2741,6 +2928,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh; (void)x_sc;const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0;@@ -2780,6 +2968,7 @@ static __device__ __forceinline__ float vec_dot_q5_1_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh; (void)x_sc;__shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];__shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1];@@ -2791,6 +2980,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh; (void)x_sc;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -2800,7 +2990,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI5_1;const int kqsx = k % QI5_1;- const block_q5_1 * bx0 = (block_q5_1 *) vx;+ const block_q5_1 * bx0 = (const block_q5_1 *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -2852,6 +3042,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh; (void)x_sc;const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1;@@ -2886,6 +3077,7 @@ static __device__ __forceinline__ float vec_dot_q8_0_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh; (void)x_sc;__shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];__shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0];@@ -2897,6 +3089,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh; (void)x_sc;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -2907,7 +3100,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kqsx = k % QI8_0;float * x_dmf = (float *) x_dm;- const block_q8_0 * bx0 = (block_q8_0 *) vx;+ const block_q8_0 * bx0 = (const block_q8_0 *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -2942,6 +3135,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh; (void)x_sc;const float * x_dmf = (const float *) x_dm;const float * y_df = (const float *) y_ds;@@ -2975,6 +3169,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh;__shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];__shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K];@@ -2988,6 +3183,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -2997,7 +3193,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI2_K;const int kqsx = k % QI2_K;- const block_q2_K * bx0 = (block_q2_K *) vx;+ const block_q2_K * bx0 = (const block_q2_K *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -3045,6 +3241,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh;const int kbx = k / QI2_K;const int ky = (k % QI2_K) * QR2_K;@@ -3118,7 +3315,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI3_K;const int kqsx = k % QI3_K;- const block_q3_K * bx0 = (block_q3_K *) vx;+ const block_q3_K * bx0 = (const block_q3_K *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -3199,7 +3396,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(const float * x_dmf = (const float *) x_dm;const float * y_df = (const float *) y_ds;- const int8_t * scales = ((int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;+ const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];@@ -3314,6 +3511,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh;__shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];__shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K];@@ -3327,6 +3525,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -3336,7 +3535,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI4_K; // == 0 if QK_K == 256const int kqsx = k % QI4_K; // == k if QK_K == 256- const block_q4_K * bx0 = (block_q4_K *) vx;+ const block_q4_K * bx0 = (const block_q4_K *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -3381,7 +3580,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI4_K/8);- const int * scales = (int *) bxi->scales;+ const int * scales = (const int *) bxi->scales;const int ksc = k % (WARP_SIZE/8);@@ -3396,6 +3595,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh;const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8);@@ -3495,6 +3695,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh;__shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];__shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K];@@ -3508,6 +3709,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -3517,7 +3719,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI5_K; // == 0 if QK_K == 256const int kqsx = k % QI5_K; // == k if QK_K == 256- const block_q5_K * bx0 = (block_q5_K *) vx;+ const block_q5_K * bx0 = (const block_q5_K *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -3573,7 +3775,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI5_K/8);- const int * scales = (int *) bxi->scales;+ const int * scales = (const int *) bxi->scales;const int ksc = k % (WARP_SIZE/8);@@ -3588,6 +3790,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh;const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);@@ -3624,6 +3827,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1(}template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ (void)x_qh;__shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];__shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K];@@ -3637,6 +3841,7 @@ template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ (void)x_qh;GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);@@ -3646,7 +3851,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinconst int kbx = k / QI6_K; // == 0 if QK_K == 256const int kqsx = k % QI6_K; // == k if QK_K == 256- const block_q6_K * bx0 = (block_q6_K *) vx;+ const block_q6_K * bx0 = (const block_q6_K *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -3708,6 +3913,7 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlinstatic __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ (void)x_qh;const float * x_dmf = (const float *) x_dm;const float * y_df = (const float *) y_ds;@@ -3750,7 +3956,7 @@ static __device__ __forceinline__ void mul_mat_q(__shared__ int tile_y_qs[mmq_x * WARP_SIZE];__shared__ half2 tile_y_ds[mmq_x * WARP_SIZE/QI8_1];- float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {0.0f};+ float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {{0.0f}};for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {@@ -4162,7 +4368,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q8_0_q8_1_mul_mat;- assert((__CUDA_ARCH__, 0));+ assert(false);#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4723,6 +4929,13 @@ static __device__ void cpy_1_f32_f16(const char * cxi, char * cdsti) {*dsti = __float2half(*xi);}+static __device__ void cpy_1_f16_f16(const char * cxi, char * cdsti) {+ const half * xi = (const half *) cxi;+ half * dsti = (half *) cdsti;++ *dsti = *xi;+}+template <cpy_kernel_t cpy_1>static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,@@ -4748,6 +4961,116 @@ static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,cpy_1(cx + x_offset, cdst + dst_offset);}+static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q8_0 * dsti = (block_q8_0 *) cdsti;++ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK8_0; j++) {+ const float v = xi[j];+ amax = fmaxf(amax, fabsf(v));+ }++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ dsti->d = d;++ for (int j = 0; j < QK8_0; ++j) {+ const float x0 = xi[j]*id;++ dsti->qs[j] = roundf(x0);+ }+}++static __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q4_0 * dsti = (block_q4_0 *) cdsti;++ float amax = 0.0f;+ float vmax = 0.0f;++ for (int j = 0; j < QK4_0; ++j) {+ const float v = xi[j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ vmax = v;+ }+ }++ const float d = vmax / -8;+ const float id = d ? 1.0f/d : 0.0f;++ dsti->d = d;++ for (int j = 0; j < QK4_0/2; ++j) {+ const float x0 = xi[0 + j]*id;+ const float x1 = xi[QK4_0/2 + j]*id;++ const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));+ const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));++ dsti->qs[j] = xi0;+ dsti->qs[j] |= xi1 << 4;+ }+}++static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q4_1 * dsti = (block_q4_1 *) cdsti;++ float vmin = FLT_MAX;+ float vmax = -FLT_MAX;++ for (int j = 0; j < QK4_1; ++j) {+ const float v = xi[j];++ if (v < vmin) vmin = v;+ if (v > vmax) vmax = v;+ }++ const float d = (vmax - vmin) / ((1 << 4) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ dsti->dm.x = d;+ dsti->dm.y = vmin;++ for (int j = 0; j < QK4_1/2; ++j) {+ const float x0 = (xi[0 + j] - vmin)*id;+ const float x1 = (xi[QK4_1/2 + j] - vmin)*id;++ const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));+ const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));++ dsti->qs[j] = xi0;+ dsti->qs[j] |= xi1 << 4;+ }+}++template <cpy_kernel_t cpy_blck, int qk>+static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,+ const int ne10, const int ne11, const int nb10, const int nb11, const int nb12) {+ const int i = (blockDim.x*blockIdx.x + threadIdx.x)*qk;++ if (i >= ne) {+ return;+ }++ const int i02 = i / (ne00*ne01);+ const int i01 = (i - i02*ne01*ne00) / ne00;+ const int i00 = (i - i02*ne01*ne00 - i01*ne00);+ const int x_offset = i00*nb00 + i01*nb01 + i02*nb02;++ const int i12 = i / (ne10*ne11);+ const int i11 = (i - i12*ne10*ne11) / ne10;+ const int i10 = (i - i12*ne10*ne11 - i11*ne10)/qk;+ const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12;++ cpy_blck(cx + x_offset, cdst + dst_offset);+}+static __device__ float rope_yarn_ramp(const float low, const float high, const int i0) {const float y = (i0 / 2 - low) / max(0.001f, high - low);return 1.0f - min(1.0f, max(0.0f, y));@@ -4902,6 +5225,65 @@ static __global__ void alibi_f32(const float * x, float * dst, const int ncols,dst[i] = col * m_k + x[i];}+static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {+ const int row = blockIdx.y;+ const int col = threadIdx.x;++ float sum = 0.0f;+ for (int i = col; i < ncols; i += blockDim.x) {+ sum += x[row * ncols + i];+ }++ sum = warp_reduce_sum(sum);++ if (col == 0) {+ dst[row] = sum;+ }+}++template<typename T>+static inline __device__ void swap(T & a, T & b) {+ T tmp = a;+ a = b;+ b = tmp;+}++template<ggml_sort_order order>+static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols) {+ // bitonic sort+ int col = threadIdx.x;+ int row = blockIdx.y;++ if (col >= ncols) return;++ const float * x_row = x + row * ncols;+ int * dst_row = dst + row * ncols;++ // initialize indices+ if (col < ncols) {+ dst_row[col] = col;+ }+ __syncthreads();++ for (int k = 2; k <= ncols; k *= 2) {+ for (int j = k / 2; j > 0; j /= 2) {+ int ixj = col ^ j;+ if (ixj > col) {+ if ((col & k) == 0) {+ if (order == GGML_SORT_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) {+ swap(dst_row[col], dst_row[ixj]);+ }+ } else {+ if (order == GGML_SORT_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) {+ swap(dst_row[col], dst_row[ixj]);+ }+ }+ }+ __syncthreads();+ }+ }+}+static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {const int col = blockDim.y*blockIdx.y + threadIdx.y;const int row = blockDim.x*blockIdx.x + threadIdx.x;@@ -4911,49 +5293,79 @@ static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int}const int i = row*ncols + col;- // dst[i] = col > n_past + row ? -INFINITY : x[i];- dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU+ //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];+ //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU+ dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;}-// the CUDA soft max implementation differs from the CPU implementation-// instead of doubles floats are used-static __global__ void soft_max_f32(const float * x, float * dst, const int ncols) {- const int row = blockDim.x*blockIdx.x + threadIdx.x;- const int block_size = blockDim.y;- const int tid = threadIdx.y;+static __global__ void soft_max_f32(const float * x, const float * y, float * dst, const int ncols, const int nrows_y, const float scale) {+ const int tid = threadIdx.x;+ const int rowx = blockIdx.x;+ const int rowy = rowx % nrows_y; // broadcast the mask (y) in the row dimension++ const int block_size = blockDim.x;++ const int warp_id = threadIdx.x / WARP_SIZE;+ const int lane_id = threadIdx.x % WARP_SIZE;++ __shared__ float buf[CUDA_SOFT_MAX_BLOCK_SIZE/WARP_SIZE];float max_val = -INFINITY;for (int col = tid; col < ncols; col += block_size) {- const int i = row*ncols + col;- max_val = max(max_val, x[i]);+ const int ix = rowx*ncols + col;+ const int iy = rowy*ncols + col;+ max_val = max(max_val, x[ix]*scale + (y ? y[iy] : 0.0f));}// find the max value in the block-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- max_val = max(max_val, __shfl_xor_sync(0xffffffff, max_val, mask, 32));+ max_val = warp_reduce_max(max_val);+ if (block_size > WARP_SIZE) {+ if (warp_id == 0) {+ buf[lane_id] = -INFINITY;+ }+ __syncthreads();++ if (lane_id == 0) {+ buf[warp_id] = max_val;+ }+ __syncthreads();++ max_val = buf[lane_id];+ max_val = warp_reduce_max(max_val);}float tmp = 0.f;for (int col = tid; col < ncols; col += block_size) {- const int i = row*ncols + col;- const float val = expf(x[i] - max_val);+ const int ix = rowx*ncols + col;+ const int iy = rowy*ncols + col;+ const float val = expf((x[ix]*scale + (y ? y[iy] : 0.0f)) - max_val);tmp += val;- dst[i] = val;+ dst[ix] = val;}- // sum up partial sums-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32);+ // find the sum of exps in the block+ tmp = warp_reduce_sum(tmp);+ if (block_size > WARP_SIZE) {+ if (warp_id == 0) {+ buf[lane_id] = 0.f;+ }+ __syncthreads();++ if (lane_id == 0) {+ buf[warp_id] = tmp;+ }+ __syncthreads();++ tmp = buf[lane_id];+ tmp = warp_reduce_sum(tmp);}const float inv_tmp = 1.f / tmp;for (int col = tid; col < ncols; col += block_size) {- const int i = row*ncols + col;+ const int i = rowx*ncols + col;dst[i] *= inv_tmp;}}@@ -4978,34 +5390,206 @@ static __global__ void clamp_f32(const float * x, float * dst, const float min,dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]);}+static __global__ void im2col_f32_f16(+ const float * x, half * dst,+ int ofs0, int ofs1, int IW, int IH, int CHW,+ int s0, int s1, int p0, int p1, int d0, int d1) {+ const int iiw = blockIdx.z * s0 + threadIdx.z * d0 - p0;+ const int iih = blockIdx.y * s1 + threadIdx.y * d1 - p1;++ const int offset_dst =+ (threadIdx.x * gridDim.y * gridDim.z + blockIdx.y * gridDim.z + blockIdx.z) * CHW ++ (blockIdx.x * (blockDim.y * blockDim.z) + threadIdx.y * blockDim.z + threadIdx.z);++ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ dst[offset_dst] = __float2half(0.0f);+ } else {+ const int offset_src = threadIdx.x * ofs0 + blockIdx.x * ofs1;+ dst[offset_dst] = __float2half(x[offset_src + iih * IW + iiw]);+ }+}+template<int qk, int qr, dequantize_kernel_t dq>-static void get_rows_cuda(const void * x, const int32_t * y, float * dst, const int nrows, const int ncols, cudaStream_t stream) {+static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {++ GGML_TENSOR_BINARY_OP_LOCALS+const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);- const int block_num_x = (ncols + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);- const dim3 block_nums(block_num_x, nrows, 1);- k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(x, y, dst, ncols);-}+ const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);++ // strides in elements+ //const size_t s0 = nb0 / ggml_element_size(dst);+ const size_t s1 = nb1 / ggml_element_size(dst);+ const size_t s2 = nb2 / ggml_element_size(dst);+ const size_t s3 = nb3 / ggml_element_size(dst);++ const size_t s10 = nb10 / ggml_element_size(src1);+ const size_t s11 = nb11 / ggml_element_size(src1);+ const size_t s12 = nb12 / ggml_element_size(src1);+ //const size_t s13 = nb13 / ggml_element_size(src1);++ GGML_ASSERT(ne00 % 2 == 0);++ k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10, ne11,*/ ne12, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);-static void add_f32_cuda(const float * x, const float * y, float * dst, const int kx, const int ky, cudaStream_t stream) {- const int num_blocks = (kx + CUDA_ADD_BLOCK_SIZE - 1) / CUDA_ADD_BLOCK_SIZE;- add_f32<<<num_blocks, CUDA_ADD_BLOCK_SIZE, 0, stream>>>(x, y, dst, kx, ky);+ (void) dst;}-static void add_f16_f32_f16_cuda(const half * x, const float * y, half * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_ADD_BLOCK_SIZE - 1) / CUDA_ADD_BLOCK_SIZE;- add_f16_f32_f16<<<num_blocks, CUDA_ADD_BLOCK_SIZE, 0, stream>>>(x, y, dst, k);-}+template<typename src0_t>+static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {-static void add_f16_f32_f32_cuda(const half * x, const float * y, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_ADD_BLOCK_SIZE - 1) / CUDA_ADD_BLOCK_SIZE;- add_f16_f32_f32<<<num_blocks, CUDA_ADD_BLOCK_SIZE, 0, stream>>>(x, y, dst, k);-}+ GGML_TENSOR_BINARY_OP_LOCALS++ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);+ const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);++ // strides in elements+ //const size_t s0 = nb0 / ggml_element_size(dst);+ const size_t s1 = nb1 / ggml_element_size(dst);+ const size_t s2 = nb2 / ggml_element_size(dst);+ const size_t s3 = nb3 / ggml_element_size(dst);++ const size_t s10 = nb10 / ggml_element_size(src1);+ const size_t s11 = nb11 / ggml_element_size(src1);+ const size_t s12 = nb12 / ggml_element_size(src1);+ //const size_t s13 = nb13 / ggml_element_size(src1);++ k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10, ne11,*/ ne12, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);-static void mul_f32_cuda(const float * x, const float * y, float * dst, const int kx, const int ky, cudaStream_t stream) {- const int num_blocks = (kx + CUDA_MUL_BLOCK_SIZE - 1) / CUDA_MUL_BLOCK_SIZE;- mul_f32<<<num_blocks, CUDA_MUL_BLOCK_SIZE, 0, stream>>>(x, y, dst, kx, ky);+ (void) dst;}+template<float (*bin_op)(const float, const float)>+struct bin_bcast_cuda {+ template<typename src0_t, typename src1_t, typename dst_t>+ void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,+ const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,+ cudaStream_t stream) {++ GGML_TENSOR_BINARY_OP_LOCALS++ int nr0 = ne10/ne0;+ int nr1 = ne11/ne1;+ int nr2 = ne12/ne2;+ int nr3 = ne13/ne3;++ int nr[4] = { nr0, nr1, nr2, nr3 };++ // collapse dimensions until first broadcast dimension+ int64_t cne0[] = {ne0, ne1, ne2, ne3};+ int64_t cne1[] = {ne10, ne11, ne12, ne13};+ size_t cnb0[] = {nb0, nb1, nb2, nb3};+ size_t cnb1[] = {nb10, nb11, nb12, nb13};+ auto collapse = [](int64_t cne[]) {+ cne[0] *= cne[1];+ cne[1] = cne[2];+ cne[2] = cne[3];+ cne[3] = 1;+ };++ auto collapse_nb = [](size_t cnb[], int64_t cne[]) {+ cnb[1] *= cne[1];+ cnb[2] *= cne[2];+ cnb[3] *= cne[3];+ };++ for (int i = 0; i < 4; i++) {+ if (nr[i] != 1) {+ break;+ }+ if (i > 0) {+ collapse_nb(cnb0, cne0);+ collapse_nb(cnb1, cne1);+ collapse(cne0);+ collapse(cne1);+ }+ }+ {+ int64_t ne0 = cne0[0];+ int64_t ne1 = cne0[1];+ int64_t ne2 = cne0[2];+ int64_t ne3 = cne0[3];++ int64_t ne10 = cne1[0];+ int64_t ne11 = cne1[1];+ int64_t ne12 = cne1[2];+ int64_t ne13 = cne1[3];++ size_t nb0 = cnb0[0];+ size_t nb1 = cnb0[1];+ size_t nb2 = cnb0[2];+ size_t nb3 = cnb0[3];++ size_t nb10 = cnb1[0];+ size_t nb11 = cnb1[1];+ size_t nb12 = cnb1[2];+ size_t nb13 = cnb1[3];++ size_t s0 = nb0 / sizeof(src1_t);+ size_t s1 = nb1 / sizeof(src1_t);+ size_t s2 = nb2 / sizeof(src1_t);+ size_t s3 = nb3 / sizeof(src1_t);++ size_t s10 = nb10 / sizeof(src1_t);+ size_t s11 = nb11 / sizeof(src1_t);+ size_t s12 = nb12 / sizeof(src1_t);+ size_t s13 = nb13 / sizeof(src1_t);++ GGML_ASSERT(s0 == 1);+ GGML_ASSERT(s10 == 1);++ const int block_size = 128;++ int64_t hne0 = std::max(ne0/2LL, 1LL);++ dim3 block_dims;+ block_dims.x = std::min<unsigned int>(hne0, block_size);+ block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);+ block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);++ dim3 block_nums(+ (hne0 + block_dims.x - 1) / block_dims.x,+ (ne1 + block_dims.y - 1) / block_dims.y,+ (ne2*ne3 + block_dims.z - 1) / block_dims.z+ );++ if (block_nums.z > 65535) {+ // this is the maximum number of blocks in z direction, fallback to 1D grid kernel+ int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;+ k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne0, ne1, ne2, ne3,+ ne10, ne11, ne12, ne13,+ /* s0, */ s1, s2, s3,+ /* s10, */ s11, s12, s13);+ } else {+ k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne0, ne1, ne2, ne3,+ ne10, ne11, ne12, ne13,+ /* s0, */ s1, s2, s3,+ /* s10, */ s11, s12, s13);+ }+ }+ }+};+static void gelu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {const int num_blocks = (k + CUDA_GELU_BLOCK_SIZE - 1) / CUDA_GELU_BLOCK_SIZE;gelu_f32<<<num_blocks, CUDA_GELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);@@ -5016,14 +5600,24 @@ static void silu_f32_cuda(const float * x, float * dst, const int k, cudaStream_silu_f32<<<num_blocks, CUDA_SILU_BLOCK_SIZE, 0, stream>>>(x, dst, k);}-static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+static void relu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;+ relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}++static void sqr_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_SQR_BLOCK_SIZE - 1) / CUDA_SQR_BLOCK_SIZE;+ sqr_f32<<<num_blocks, CUDA_SQR_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}++static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {GGML_ASSERT(ncols % WARP_SIZE == 0);if (ncols < 1024) {const dim3 block_dims(WARP_SIZE, 1, 1);- norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols);+ norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);} else {const dim3 block_dims(1024, 1, 1);- norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols);+ norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);}}@@ -5045,34 +5639,10 @@ static void quantize_row_q8_1_cuda(const float * x, void * vy, const int kx, conquantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);}-template<typename dst_t>-static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- dequantize_block<QK4_0, QR4_0, dequantize_q4_0><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}--template<typename dst_t>-static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- dequantize_block<QK4_1, QR4_1, dequantize_q4_1><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}--template<typename dst_t>-static void dequantize_row_q5_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- dequantize_block<QK5_0, QR5_0, dequantize_q5_0><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}--template<typename dst_t>-static void dequantize_row_q5_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- dequantize_block<QK5_1, QR5_1, dequantize_q5_1><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}--template<typename dst_t>-static void dequantize_row_q8_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- dequantize_block<QK8_0, QR8_0, dequantize_q8_0><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);+ dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);}template<typename dst_t>@@ -5121,6 +5691,64 @@ static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int k, cu#endif}+static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;+ case GGML_TYPE_Q4_1:+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_Q2_K:+ return dequantize_row_q2_K_cuda;+ case GGML_TYPE_Q3_K:+ return dequantize_row_q3_K_cuda;+ case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;+ case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_F32:+ return dequantize_block_cuda<1, 1, convert_f32>;+ default:+ return nullptr;+ }+}++static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;+ case GGML_TYPE_Q4_1:+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_Q2_K:+ return dequantize_row_q2_K_cuda;+ case GGML_TYPE_Q3_K:+ return dequantize_row_q3_K_cuda;+ case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;+ case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_F16:+ return dequantize_block_cuda<1, 1, convert_f16>;+ default:+ return nullptr;+ }+}+static void dequantize_mul_mat_vec_q4_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;@@ -5209,6 +5837,15 @@ static void dequantize_mul_mat_vec_q6_K_cuda(const void * vx, const float * y, fdequantize_mul_mat_vec_q6_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}+static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<1, 1, convert_f16>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}+static void mul_mat_vec_q4_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {GGML_ASSERT(ncols % QK4_0 == 0);const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;@@ -5299,90 +5936,13 @@ static void mul_mat_vec_q6_K_q8_1_cuda(const void * vx, const void * vy, float *<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);}-static void convert_fp16_to_fp32_cuda(const void * vx, float * y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- dequantize_block<1, 1, convert_f16><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}--static void convert_fp32_to_fp16_cuda(const void * vx, half * y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;- dequantize_block<1, 1, convert_f32><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}+static void ggml_mul_mat_q4_0_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<1, 1, convert_f16>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}--static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- return dequantize_row_q4_0_cuda;- case GGML_TYPE_Q4_1:- return dequantize_row_q4_1_cuda;- case GGML_TYPE_Q5_0:- return dequantize_row_q5_0_cuda;- case GGML_TYPE_Q5_1:- return dequantize_row_q5_1_cuda;- case GGML_TYPE_Q8_0:- return dequantize_row_q8_0_cuda;- case GGML_TYPE_Q2_K:- return dequantize_row_q2_K_cuda;- case GGML_TYPE_Q3_K:- return dequantize_row_q3_K_cuda;- case GGML_TYPE_Q4_K:- return dequantize_row_q4_K_cuda;- case GGML_TYPE_Q5_K:- return dequantize_row_q5_K_cuda;- case GGML_TYPE_Q6_K:- return dequantize_row_q6_K_cuda;- case GGML_TYPE_F32:- return convert_fp32_to_fp16_cuda;- default:- return nullptr;- }-}--static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- return dequantize_row_q4_0_cuda;- case GGML_TYPE_Q4_1:- return dequantize_row_q4_1_cuda;- case GGML_TYPE_Q5_0:- return dequantize_row_q5_0_cuda;- case GGML_TYPE_Q5_1:- return dequantize_row_q5_1_cuda;- case GGML_TYPE_Q8_0:- return dequantize_row_q8_0_cuda;- case GGML_TYPE_Q2_K:- return dequantize_row_q2_K_cuda;- case GGML_TYPE_Q3_K:- return dequantize_row_q3_K_cuda;- case GGML_TYPE_Q4_K:- return dequantize_row_q4_K_cuda;- case GGML_TYPE_Q5_K:- return dequantize_row_q5_K_cuda;- case GGML_TYPE_Q6_K:- return dequantize_row_q6_K_cuda;- case GGML_TYPE_F16:- return convert_fp16_to_fp32_cuda;- default:- return nullptr;- }-}--static void ggml_mul_mat_q4_0_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = g_compute_capabilities[id];int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -5868,6 +6428,49 @@ static void ggml_cpy_f32_f16_cuda((cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);}+static void ggml_cpy_f32_q8_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,+ const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {++ GGML_ASSERT(ne % QK8_0 == 0);+ const int num_blocks = ne / QK8_0;+ cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+}++static void ggml_cpy_f32_q4_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,+ const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {++ GGML_ASSERT(ne % QK4_0 == 0);+ const int num_blocks = ne / QK4_0;+ cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+}++static void ggml_cpy_f32_q4_1_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,+ const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {++ GGML_ASSERT(ne % QK4_1 == 0);+ const int num_blocks = ne / QK4_1;+ cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+}++static void ggml_cpy_f16_f16_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,+ const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {++ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f16_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+}+static void scale_f32_cuda(const float * x, float * dst, const float scale, const int k, cudaStream_t stream) {const int num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;scale_f32<<<num_blocks, CUDA_SCALE_BLOCK_SIZE, 0, stream>>>(x, dst, scale, k);@@ -5944,6 +6547,27 @@ static void alibi_f32_cuda(const float * x, float * dst, const int ncols, constalibi_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, k_rows, n_heads_log2_floor, m0, m1);}+static void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ const dim3 block_nums(1, nrows, 1);+ k_sum_rows_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);+}++static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {+ // bitonic sort requires ncols to be power of 2+ GGML_ASSERT((ncols & (ncols - 1)) == 0);++ const dim3 block_dims(ncols, 1, 1);+ const dim3 block_nums(1, nrows, 1);+ if (order == GGML_SORT_ASC) {+ k_argsort_f32_i32<GGML_SORT_ASC><<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);+ } else if (order == GGML_SORT_DESC) {+ k_argsort_f32_i32<GGML_SORT_DESC><<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);+ } else {+ GGML_ASSERT(false);+ }+}+static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;@@ -5951,10 +6575,21 @@ static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncolsdiag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);}-static void soft_max_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, cudaStream_t stream) {- const dim3 block_dims(1, WARP_SIZE, 1);+static void soft_max_f32_cuda(const float * x, const float * y, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, cudaStream_t stream) {+ int nth = WARP_SIZE;+ while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;+ const dim3 block_dims(nth, 1, 1);const dim3 block_nums(nrows_x, 1, 1);- soft_max_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x);+ soft_max_f32<<<block_nums, block_dims, 0, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+}++static void im2col_f32_f16_cuda(const float * x, half * dst,+ int OH, int IW, int IH, int OW, int IC,+ int KH, int KW, int N, int ofs0, int ofs1,+ int s0, int s1, int p0, int p1, int d0, int d1, cudaStream_t stream) {+ dim3 block_nums(IC, OH, OW);+ dim3 block_dims(N, KH, KW);+ im2col_f32_f16<<<block_nums, block_dims, 0, stream>>>(x, dst, ofs0, ofs1, IW, IH, (IC * KH * KW), s0, s1, p0, p1, d0, d1);}// buffer pool for cuda@@ -6115,11 +6750,11 @@ void ggml_init_cublas() {CUDA_CHECK(cudaStreamCreateWithFlags(&g_cudaStreams[id][is], cudaStreamNonBlocking));}-#if !defined(GGML_USE_TINYBLAS)+#ifndef GGML_USE_TINYBLAS// create cublas handleCUBLAS_CHECK(cublasCreate(&g_cublas_handles[id]));CUBLAS_CHECK(cublasSetMathMode(g_cublas_handles[id], CUBLAS_TF32_TENSOR_OP_MATH));-#endif // !defined(GGML_USE_TINYBLAS)+#endif}// configure logging to stdout@@ -6224,99 +6859,40 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(}}-static void ggml_cuda_op_repeat(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_d, const float * src1_d, float * dst_d, const cudaStream_t & stream) {- // guaranteed to be an integer due to the check in ggml_can_repeat- const int64_t ne0 = dst->ne[0];- const int64_t ne1 = dst->ne[1];- const int64_t ne2 = dst->ne[2];- const int64_t ne3 = dst->ne[3];-- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t ne03 = src0->ne[3];-- const size_t nb0 = dst->nb[0];- const size_t nb1 = dst->nb[1];- const size_t nb2 = dst->nb[2];- const size_t nb3 = dst->nb[3];-- const size_t nb00 = src0->nb[0];- const size_t nb01 = src0->nb[1];- const size_t nb02 = src0->nb[2];- const size_t nb03 = src0->nb[3];-- const int nr0 = (int)(ne0/ne00);- const int nr1 = (int)(ne1/ne01);- const int nr2 = (int)(ne2/ne02);- const int nr3 = (int)(ne3/ne03);-- // TODO: support for transposed / permuted tensors- GGML_ASSERT(nb0 == sizeof(float));- GGML_ASSERT(nb00 == sizeof(float));-- // TODO: very inefficient, implement in a kernel, or fewer cudaMemcpyAsync calls for contiguous tensors- for (int i3 = 0; i3 < nr3; i3++) {- for (int k3 = 0; k3 < ne03; k3++) {- for (int i2 = 0; i2 < nr2; i2++) {- for (int k2 = 0; k2 < ne02; k2++) {- for (int i1 = 0; i1 < nr1; i1++) {- for (int k1 = 0; k1 < ne01; k1++) {- for (int i0 = 0; i0 < nr0; i0++) {- CUDA_CHECK(cudaMemcpyAsync(- (char *) dst_d + (i3*ne03 + k3)*nb3 + (i2*ne02 + k2)*nb2 + (i1*ne01 + k1)*nb1 + (i0*ne00)*nb0,- (const char *) src0_d + ( k3)*nb03 + ( k2)*nb02 + ( k1)*nb01,- ne00*nb0, cudaMemcpyDeviceToDevice, stream));- }- }- }- }- }- }- }-- (void) src1;- (void) src1_d;-}-static void ggml_cuda_op_get_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_d, const float * src1_d, float * dst_d, const cudaStream_t & stream) {GGML_ASSERT(src1->type == GGML_TYPE_I32);GGML_ASSERT(dst->type == GGML_TYPE_F32);- GGML_ASSERT(ggml_is_contiguous(src0));- GGML_ASSERT(ggml_is_contiguous(src1));- GGML_ASSERT(ggml_is_contiguous(dst));- const int ncols = src0->ne[0];- const int nrows = ggml_nelements(src1);+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));+ GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));+ GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));const int32_t * src1_i32 = (const int32_t *) src1_d;switch (src0->type) {case GGML_TYPE_F16:- get_rows_cuda<1, 1, convert_f16>(src0_d, src1_i32, dst_d, nrows, ncols, stream);+ get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);break;case GGML_TYPE_F32:- get_rows_cuda<1, 1, convert_f32>(src0_d, src1_i32, dst_d, nrows, ncols, stream);+ get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);break;case GGML_TYPE_Q4_0:- get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0_d, src1_i32, dst_d, nrows, ncols, stream);+ get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);break;case GGML_TYPE_Q4_1:- get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0_d, src1_i32, dst_d, nrows, ncols, stream);+ get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);break;case GGML_TYPE_Q5_0:- get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0_d, src1_i32, dst_d, nrows, ncols, stream);+ get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);break;case GGML_TYPE_Q5_1:- get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0_d, src1_i32, dst_d, nrows, ncols, stream);+ get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);break;case GGML_TYPE_Q8_0:- get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0_d, src1_i32, dst_d, nrows, ncols, stream);+ get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);break;default:// TODO: k-quants@@ -6325,44 +6901,55 @@ static void ggml_cuda_op_get_rows(}}-inline void ggml_cuda_op_add(+template<class op>+inline void ggml_cuda_op_bin_bcast(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {GGML_ASSERT(src1->type == GGML_TYPE_F32);- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];-if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- add_f32_cuda(src0_dd, src1_dd, dst_dd, ggml_nelements(src0), ne10*ne11, main_stream);+ op()(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);} else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {- add_f16_f32_f16_cuda((const half *) src0_dd, src1_dd, (half *) dst_dd, ggml_nelements(src0), main_stream);+ op()(src0, src1, dst, (const half *) src0_dd, src1_dd, (half *) dst_dd, main_stream);} else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {- add_f16_f32_f32_cuda((const half *) src0_dd, src1_dd, dst_dd, ggml_nelements(src0), main_stream);+ op()(src0, src1, dst, (const half *) src0_dd, src1_dd, dst_dd, main_stream);} else {- fprintf(stderr, "src0->type: %d dst->type: %d\n", src0->type, dst->type);+ fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,+ ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));GGML_ASSERT(false);}+}++static void ggml_cuda_op_repeat(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_d, const float * src1_d, float * dst_d, const cudaStream_t & main_stream) {++ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, src0, dst, nullptr, src0_d, dst_d, main_stream);(void) src1;- (void) dst;+ (void) src1_d;}-inline void ggml_cuda_op_mul(+inline void ggml_cuda_op_add(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+}- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];+inline void ggml_cuda_op_mul(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {- mul_f32_cuda(src0_dd, src1_dd, dst_dd, ggml_nelements(src0), ne10*ne11, main_stream);+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+}- (void) dst;+inline void ggml_cuda_op_div(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {++ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);}inline void ggml_cuda_op_gelu(@@ -6393,6 +6980,34 @@ inline void ggml_cuda_op_silu((void) src1_dd;}+inline void ggml_cuda_op_relu(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ relu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);++ (void) src1;+ (void) dst;+ (void) src1_dd;+}++inline void ggml_cuda_op_sqr(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ sqr_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);++ (void) src1;+ (void) dst;+ (void) src1_dd;+}+inline void ggml_cuda_op_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {@@ -6403,7 +7018,10 @@ inline void ggml_cuda_op_norm(const int64_t ne00 = src0->ne[0];const int64_t nrows = ggml_nrows(src0);- norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, main_stream);+ float eps;+ memcpy(&eps, dst->op_params, sizeof(float));++ norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, eps, main_stream);(void) src1;(void) dst;@@ -6558,6 +7176,8 @@ inline void ggml_cuda_op_mul_mat_vec_q(const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,const int64_t src1_padded_row_size, const cudaStream_t & stream) {+ GGML_ASSERT(ggml_nrows(src1) == 1);+const int64_t ne00 = src0->ne[0];const int64_t row_diff = row_high - row_low;@@ -6617,7 +7237,8 @@ inline void ggml_cuda_op_dequantize_mul_mat_vec(size_t ash;dfloat * src1_dfloat = nullptr; // dfloat == half- bool src1_convert_f16 = src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||+ bool src1_convert_f16 =+ src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;@@ -6731,8 +7352,7 @@ inline void ggml_cuda_op_mul_mat_cublas(src1_as_f16 = (half *) ggml_cuda_pool_malloc(ne * sizeof(half), &src1_as);to_fp16_cuda(src1_ddf_i, src1_as_f16, ne, stream);}- const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddq_i : src1_as_f16;-+ const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16;size_t dst_as = 0;half * dst_f16 = (half *) ggml_cuda_pool_malloc(row_diff*src1_ncols * sizeof(half), &dst_as);@@ -6906,6 +7526,81 @@ inline void ggml_cuda_op_alibi((void) src1_dd;}+inline void ggml_cuda_op_im2col(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F16);++ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t*)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t*)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t*)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t*)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t*)(dst->op_params))[5];++ const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;++ const int64_t N = src1->ne[is_2D ? 3 : 2];+ const int64_t IC = src1->ne[is_2D ? 2 : 1];+ const int64_t IH = is_2D ? src1->ne[1] : 1;+ const int64_t IW = src1->ne[0];++ const int64_t KH = is_2D ? src0->ne[1] : 1;+ const int64_t KW = src0->ne[0];++ const int64_t OH = is_2D ? dst->ne[2] : 1;+ const int64_t OW = dst->ne[1];++ const size_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4; // nb is byte offset, src is type float32+ const size_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32++ im2col_f32_f16_cuda(src1_dd, (half*) dst_dd,+ OH, IW, IH, OW, IC, KH, KW, N,+ ofs0, ofs1, s0, s1, p0, p1, d0, d1, main_stream);++ (void) src0;+ (void) src0_dd;+}++inline void ggml_cuda_op_sum_rows(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ const int64_t ncols = src0->ne[0];+ const int64_t nrows = ggml_nrows(src0);++ sum_rows_f32_cuda(src0_dd, dst_dd, ncols, nrows, main_stream);++ (void) src1;+ (void) dst;+ (void) src1_dd;+}++inline void ggml_cuda_op_argsort(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_I32);++ const int64_t ncols = src0->ne[0];+ const int64_t nrows = ggml_nrows(src0);++ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];++ argsort_f32_i32_cuda(src0_dd, (int *)dst_dd, ncols, nrows, order, main_stream);++ (void) src1;+ (void) dst;+ (void) src1_dd;+}+inline void ggml_cuda_op_diag_mask_inf(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {@@ -6933,14 +7628,18 @@ inline void ggml_cuda_op_soft_max(GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional+const int64_t ne00 = src0->ne[0];- const int64_t nrows = ggml_nrows(src0);+ const int64_t nrows_x = ggml_nrows(src0);+ const int64_t nrows_y = src1 ? ggml_nrows(src1) : 1;- soft_max_f32_cuda(src0_dd, dst_dd, ne00, nrows, main_stream);+ float scale = 1.0f;+ memcpy(&scale, dst->op_params, sizeof(float));++ soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream);- (void) src1;(void) dst;- (void) src1_dd;}inline void ggml_cuda_op_scale(@@ -7146,10 +7845,9 @@ static void ggml_cuda_op_mul_mat(const bool src0_on_device = src0->backend == GGML_BACKEND_GPU || src0->backend == GGML_BACKEND_GPU_SPLIT;const bool src0_is_contiguous = ggml_is_contiguous(src0);-const bool src1_is_contiguous = ggml_is_contiguous(src1);- const int64_t src1_padded_col_size = ne10 % MATRIX_ROW_PADDING == 0 ?- ne10 : ne10 - ne10 % MATRIX_ROW_PADDING + MATRIX_ROW_PADDING;++ const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING);const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT;GGML_ASSERT(!(split && ne02 > 1));@@ -7211,7 +7909,7 @@ static void ggml_cuda_op_mul_mat(if (src0_on_device && src0_is_contiguous) {src0_dd[id] = (char *) src0_extra->data_device[id];} else {- const size_t size_src0_ddq = split ? (row_high[id]-row_low[id])*ne00 * src0_ts/src0_bs : ggml_nbytes(src0);+ // const size_t size_src0_ddq = split ? (row_high[id]-row_low[id])*ne00 * src0_ts/src0_bs : ggml_nbytes(src0);src0_dd[id] = (char *) ggml_cuda_pool_malloc(ggml_nbytes(src0), &src0_as[id]);}@@ -7274,7 +7972,7 @@ static void ggml_cuda_op_mul_mat(const size_t src1_ddq_i_offset = (i0*ne11 + src1_col_0) * src1_padded_col_size*q8_1_ts/q8_1_bs;// for split tensors the data begins at i0 == i0_offset_low- char * src0_dd_i = src0_dd[id] + (i0/i02_divisor) * ne01*ne00*src0_ts/src0_bs;+ char * src0_dd_i = src0_dd[id] + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;float * src1_ddf_i = src1_ddf[id] + (i0*ne11 + src1_col_0) * ne10;char * src1_ddq_i = src1_ddq[id] + src1_ddq_i_offset;float * dst_dd_i = dst_dd[id] + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);@@ -7419,6 +8117,10 @@ static void ggml_cuda_mul(const ggml_tensor * src0, const ggml_tensor * src1, ggggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_mul);}+static void ggml_cuda_div(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_div);+}+static void ggml_cuda_gelu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_gelu);}@@ -7427,6 +8129,14 @@ static void ggml_cuda_silu(const ggml_tensor * src0, const ggml_tensor * src1, gggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_silu);}+static void ggml_cuda_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_relu);+}++static void ggml_cuda_sqr(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_sqr);+}+static void ggml_cuda_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_norm);}@@ -7514,7 +8224,7 @@ static void ggml_cuda_mul_mat_vec_nc(const ggml_tensor * src0, const ggml_tensorggml_mul_mat_vec_nc_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, row_stride_x, ne02, ne12, channel_stride_x, main_stream);}-__global__ void k_compute_batched_ptrs(+static __global__ void k_compute_batched_ptrs(const half * src0_as_f16, const half * src1_as_f16, half * dst_f16,const void ** ptrs_src, void ** ptrs_dst,int ne12, int ne13,@@ -7571,9 +8281,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constCUDA_CHECK(ggml_cuda_set_device(g_main_device));cudaStream_t main_stream = g_cudaStreams[g_main_device][0];- int id;- CUDA_CHECK(cudaGetDevice(&id));- CUBLAS_SET_STREAM(id, main_stream);+ CUBLAS_SET_STREAM(g_main_device, main_stream);ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;void * src0_ddq = src0_extra->data_device[g_main_device];@@ -7630,7 +8338,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const// there is no broadcast and src0, src1 are contiguous across dims 2, 3// use cublasGemmStridedBatchedExCUBLAS_CHECK(- cublasGemmStridedBatchedEx(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,+ cublasGemmStridedBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,ne01, ne11, ne10,&alpha_f16, (const char *) src0_as_f16, CUDA_R_16F, nb01/sizeof(half), src0->nb[2]/sizeof(half), // strideA(const char *) src1_as_f16, CUDA_R_16F, nb11/sizeof(float), src1->nb[2]/sizeof(float), // strideB@@ -7664,7 +8372,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constCUDA_CHECK(cudaGetLastError());CUBLAS_CHECK(- cublasGemmBatchedEx(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,+ cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,ne01, ne11, ne10,&alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, nb01/sizeof(half),(const void **) (ptrs_src + 1*ne23), CUDA_R_16F, nb11/sizeof(float),@@ -7734,10 +8442,11 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1#ifdef GGML_CUDA_FORCE_DMMVconst bool use_mul_mat_vec_q = false;#else- const bool use_mul_mat_vec_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type);+ const bool use_mul_mat_vec_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type) && ggml_nrows(src1) == 1;#endif // GGML_CUDA_FORCE_DMMVif (use_mul_mat_vec_q) {+ // NOTE: this kernel does not support ggml_nrows(src1) > 1ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true);} else {ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false);@@ -7762,22 +8471,268 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1}}-static void ggml_cuda_scale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_scale);-}+#if 0+template<typename ... Srcs>+static __global__ void k_compute_batched_ptrs_id(+ const void ** ptrs_src, void ** ptrs_dst,+ int ne12, int ne13,+ int ne23,+ int nb02, int nb03,+ int nb12, int nb13,+ int nb2, int nb3,+ int r2, int r3,+ ggml_type src0_type, half * src0_as_f16, int64_t src0_ne,+ const half * src1_f16, half * dst_f16,+ const int32_t * ids, const int id,+ Srcs... src0s) {++ int i = ids[id];++ half * src0_f16;+ const void * srcs_ar[] = { (const half *) src0s... };+ if (src0_type == GGML_TYPE_F16) {+ src0_f16 = (half *) srcs_ar[i];+ } else {+ src0_f16 = src0_as_f16;+ if (threadIdx.x == 0 && threadIdx.y == 0) {+ const to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(src0_type);+ to_fp16(srcs_ar[i], src0_f16, src0_ne, cudaStreamFireAndForget);+ }+ }-static void ggml_cuda_clamp(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_clamp);-}+ int i13 = blockIdx.x * blockDim.x + threadIdx.x;+ int i12 = blockIdx.y * blockDim.y + threadIdx.y;-static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- const int64_t ne = ggml_nelements(src0);- GGML_ASSERT(ne == ggml_nelements(src1));+ if (i13 >= ne13 || i12 >= ne12) {+ return;+ }- GGML_ASSERT(src0->backend == GGML_BACKEND_GPU);- GGML_ASSERT(src1->backend == GGML_BACKEND_GPU);+ int i03 = i13 / r3;+ int i02 = i12 / r2;- GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX);+ ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_f16 + i02*nb02 + i03*nb03;+ ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_f16 + i12*nb12/2 + i13*nb13/2;+ ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst_f16 + i12* nb2/2 + i13* nb3/2;+}++static void ggml_cuda_mul_mat_id_cublas(ggml_tensor * dst) {+ const struct ggml_tensor * ids = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+ const struct ggml_tensor * src00 = dst->src[2];++ const int id = dst->op_params[0];++ GGML_ASSERT(!ggml_is_transposed(src00));+ GGML_ASSERT(!ggml_is_transposed(src1));++ GGML_ASSERT(src00->backend != GGML_BACKEND_GPU_SPLIT);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);++ const int64_t ne00 = src00->ne[0]; GGML_UNUSED(ne00);+ const int64_t ne01 = src00->ne[1];+ const int64_t ne02 = src00->ne[2];+ const int64_t ne03 = src00->ne[3];++ //const int64_t nb01 = src00->nb[1];+ const int64_t nb02 = src00->nb[2]; GGML_UNUSED(nb02);+ const int64_t nb03 = src00->nb[3]; GGML_UNUSED(nb03);++ const int64_t ne10 = src1->ne[0];+ const int64_t ne11 = src1->ne[1];+ const int64_t ne12 = src1->ne[2];+ const int64_t ne13 = src1->ne[3];++ //const int64_t nb11 = src1->nb[1];+ const int64_t nb12 = src1->nb[2]; GGML_UNUSED(nb12);+ const int64_t nb13 = src1->nb[3]; GGML_UNUSED(nb13);++ const int64_t ne1 = ggml_nelements(src1);+ const int64_t ne = ggml_nelements(dst);++ CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ cudaStream_t main_stream = g_cudaStreams[g_main_device][0];++ CUBLAS_SET_STREAM(id, main_stream);++ //ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;+ //void * src0_ddq = src0_extra->data_device[g_main_device];+ //half * src0_as_f16 = (half *) src0_ddq;++ ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;+ float * src1_ddf = (float *) src1_extra->data_device[g_main_device];++ ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;+ float * dst_ddf = (float *) dst_extra->data_device[g_main_device];++ // convert src1 to fp16+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ GGML_ASSERT(to_fp16_cuda != nullptr);++ size_t src1_as = 0;+ half * src1_as_f16 = (half *) ggml_cuda_pool_malloc(ne1 * sizeof(half), &src1_as);+ to_fp16_cuda(src1_ddf, src1_as_f16, ne1, main_stream);++ size_t dst_as = 0;+ half * dst_f16 = (half *) ggml_cuda_pool_malloc(ne * sizeof(half), &dst_as);++ GGML_ASSERT(ne12 % ne02 == 0);+ GGML_ASSERT(ne13 % ne03 == 0);++ // broadcast factors+ const int64_t r2 = ne12/ne02;+ const int64_t r3 = ne13/ne03;++ const half alpha_f16 = 1.0f;+ const half beta_f16 = 0.0f;++ // use cublasGemmBatchedEx+ const int ne23 = ne12*ne13;++ const void ** ptrs_src = nullptr;+ void ** ptrs_dst = nullptr;++ size_t ptrs_src_s = 0;+ size_t ptrs_dst_s = 0;++ ptrs_src = (const void **) ggml_cuda_pool_malloc(2*ne23*sizeof(void *), &ptrs_src_s);+ ptrs_dst = ( void **) ggml_cuda_pool_malloc(1*ne23*sizeof(void *), &ptrs_dst_s);++ int64_t src0_ne = ggml_nelements(src00);+ half * src0_as_f16 = nullptr;+ size_t src0_as = 0;+ if (src00->type != GGML_TYPE_F16) {+ src0_as_f16 = (half *) ggml_cuda_pool_malloc(src0_ne * sizeof(half), &src0_as);+ }++ static_assert(GGML_MAX_SRC == 6, "GGML_MAX_SRC == 6");+ dim3 block_dims(ne13, ne12);+ k_compute_batched_ptrs_id<<<1, block_dims, 0, main_stream>>>(+ ptrs_src, ptrs_dst,+ ne12, ne13,+ ne23,+ ne00*ne01*sizeof(half), ne00*ne01*ne02*sizeof(half),+ nb12, nb13,+ dst->nb[2], dst->nb[3],+ r2, r3,+ src00->type, src0_as_f16, src0_ne,+ src1_as_f16, dst_f16,+ (const int *)((ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device], id,+ dst->src[2] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[2]->extra)->data_device[g_main_device] : nullptr,+ dst->src[3] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[3]->extra)->data_device[g_main_device] : nullptr,+ dst->src[4] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[4]->extra)->data_device[g_main_device] : nullptr,+ dst->src[5] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[5]->extra)->data_device[g_main_device] : nullptr+ );+ CUDA_CHECK(cudaGetLastError());++ CUBLAS_CHECK(+ cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,+ ne01, ne11, ne10,+ &alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, ne00,+ (const void **) (ptrs_src + 1*ne23), CUDA_R_16F, ne10,+ &beta_f16, ( void **) (ptrs_dst + 0*ne23), CUDA_R_16F, ne01,+ ne23,+ CUBLAS_COMPUTE_16F,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));++ if (src0_as != 0) {+ ggml_cuda_pool_free(src0_as_f16, src0_as);+ }+ if (ptrs_src_s != 0) {+ ggml_cuda_pool_free(ptrs_src, ptrs_src_s);+ }+ if (ptrs_dst_s != 0) {+ ggml_cuda_pool_free(ptrs_dst, ptrs_dst_s);+ }++ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);+ to_fp32_cuda(dst_f16, dst_ddf, ne, main_stream);++ ggml_cuda_pool_free(src1_as_f16, src1_as);+ ggml_cuda_pool_free(dst_f16, dst_as);+}+#endif++static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+#if 0+ ggml_cuda_mul_mat_id_cublas(dst);+ // TODO: mmq/mmv support+#endif++ GGML_ASSERT(dst->backend == GGML_BACKEND_GPU);++ const struct ggml_tensor * ids = src0;+ const int32_t id = ((int32_t *) dst->op_params)[0];+ const int32_t n_as = ((int32_t *) dst->op_params)[1];++ std::vector<char> ids_host(ggml_nbytes(ids));++ if (ids->backend == GGML_BACKEND_GPU) {+ const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device];+ CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, g_cudaStreams[g_main_device][0]));+ CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[g_main_device][0]));+ } else {+ memcpy(ids_host.data(), ids->data, ggml_nbytes(ids));+ }++ const ggml_tensor_extra_gpu * src1_extra = (const ggml_tensor_extra_gpu *) src1->extra;+ const ggml_tensor_extra_gpu * dst_extra = (const ggml_tensor_extra_gpu *) dst->extra;++ ggml_tensor_extra_gpu src1_row_extra;+ ggml_tensor_extra_gpu dst_row_extra;++ ggml_tensor src1_row = *src1;+ ggml_tensor dst_row = *dst;++ src1_row.ne[1] = 1;+ dst_row.ne[1] = 1;++ src1_row.nb[2] = src1_row.nb[1];+ dst_row.nb[2] = dst_row.nb[1];++ src1_row.nb[3] = src1_row.nb[1];+ dst_row.nb[3] = dst_row.nb[1];++ src1_row.extra = &src1_row_extra;+ dst_row.extra = &dst_row_extra;+++ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ //int32_t row_id;+ //CUDA_CHECK(cudaMemcpyAsync(&row_id, ids_dev + i01*ids->nb[1] + id*ids->nb[0], sizeof(int32_t), cudaMemcpyDeviceToHost, g_cudaStreams[g_main_device][0]));+ //CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[g_main_device][0]));++ const int32_t row_id = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);++ GGML_ASSERT(row_id >= 0 && row_id < n_as);++ const struct ggml_tensor * src0_row = dst->src[row_id + 2];++ src1_row_extra.data_device[g_main_device] = (char *) src1_extra->data_device[g_main_device] + i01*src1->nb[1];+ src1_row.data = (char *) src1->data + i01*src1->nb[1];++ dst_row_extra.data_device[g_main_device] = (char *) dst_extra->data_device[g_main_device] + i01*dst->nb[1];+ dst_row.data = (char *) dst->data + i01*dst->nb[1];++ ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row);+ }+}++static void ggml_cuda_scale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_scale);+}++static void ggml_cuda_clamp(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_clamp);+}++static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ const int64_t ne = ggml_nelements(src0);+ GGML_ASSERT(ne == ggml_nelements(src1));++ GGML_ASSERT(src0->backend == GGML_BACKEND_GPU);+ GGML_ASSERT(src1->backend == GGML_BACKEND_GPU);++ GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX);GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX);const int64_t ne00 = src0->ne[0];@@ -7806,14 +8761,20 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggchar * src1_ddc = (char *) src1_extra->data_device[g_main_device];if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {- ggml_cpy_f32_f32_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02,- ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {- ggml_cpy_f32_f16_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02,- ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {+ ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {+ ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {+ ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {+ ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);} else {- fprintf(stderr, "%s: unsupported type combination (%d to %d)\n", __func__,- src0->type, src1->type);+ fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,+ ggml_type_name(src0->type), ggml_type_name(src1->type));GGML_ASSERT(false);}@@ -7821,6 +8782,7 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, gg}static void ggml_cuda_dup(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ // TODO: why do we pass dst as src1 here?ggml_cuda_cpy(src0, dst, nullptr);(void) src1;}@@ -7842,6 +8804,20 @@ static void ggml_cuda_alibi(const ggml_tensor * src0, const ggml_tensor * src1,ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_alibi);}+static void ggml_cuda_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_im2col);+}++static void ggml_cuda_sum_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_ASSERT(ggml_is_contiguous(src0));+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_sum_rows);+}++static void ggml_cuda_argsort(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_ASSERT(ggml_is_contiguous(src0));+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_argsort);+}+static void ggml_cuda_nop(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {(void) src0;(void) src1;@@ -7953,11 +8929,11 @@ static size_t g_temp_tensor_extra_index = 0;static ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() {if (g_temp_tensor_extras == nullptr) {- g_temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_MAX_NODES];+ g_temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES];}size_t alloc_index = g_temp_tensor_extra_index;- g_temp_tensor_extra_index = (g_temp_tensor_extra_index + 1) % GGML_MAX_NODES;+ g_temp_tensor_extra_index = (g_temp_tensor_extra_index + 1) % GGML_CUDA_MAX_NODES;ggml_tensor_extra_gpu * extra = &g_temp_tensor_extras[alloc_index];memset(extra, 0, sizeof(*extra));@@ -8097,8 +9073,9 @@ void ggml_cuda_set_main_device(const int main_device) {main_device, g_device_count, g_main_device);return;}- g_main_device = main_device;- if (g_device_count > 1) {++ if (g_main_device != main_device && g_device_count > 1) {+ g_main_device = main_device;cudaDeviceProp prop;CUDA_CHECK(cudaGetDeviceProperties(&prop, g_main_device));fprintf(stderr, "%s: using device %d (%s) as main device\n", __func__, g_main_device, prop.name);@@ -8135,6 +9112,15 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_return false;}+ if (tensor->op == GGML_OP_MUL_MAT) {+ if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) {+#ifndef NDEBUG+ fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = " PRId64 ", src1->ne[3] = " PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]);+#endif+ return false;+ }+ }+switch (tensor->op) {case GGML_OP_REPEAT:func = ggml_cuda_repeat;@@ -8151,6 +9137,9 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_case GGML_OP_MUL:func = ggml_cuda_mul;break;+ case GGML_OP_DIV:+ func = ggml_cuda_div;+ break;case GGML_OP_UNARY:switch (ggml_get_unary_op(tensor)) {case GGML_UNARY_OP_GELU:@@ -8159,9 +9148,13 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_case GGML_UNARY_OP_SILU:func = ggml_cuda_silu;break;+ case GGML_UNARY_OP_RELU:+ func = ggml_cuda_relu;+ break;default:return false;- } break;+ }+ break;case GGML_OP_NORM:func = ggml_cuda_norm;break;@@ -8174,9 +9167,18 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_}func = ggml_cuda_mul_mat;break;+ case GGML_OP_MUL_MAT_ID:+ if (!any_on_device && !ggml_cuda_can_mul_mat(tensor->src[2], tensor->src[1], tensor)) {+ return false;+ }+ func = ggml_cuda_mul_mat_id;+ break;case GGML_OP_SCALE:func = ggml_cuda_scale;break;+ case GGML_OP_SQR:+ func = ggml_cuda_sqr;+ break;case GGML_OP_CLAMP:if (!any_on_device) {return false;@@ -8207,6 +9209,15 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_case GGML_OP_ALIBI:func = ggml_cuda_alibi;break;+ case GGML_OP_IM2COL:+ func = ggml_cuda_im2col;+ break;+ case GGML_OP_SUM_ROWS:+ func = ggml_cuda_sum_rows;+ break;+ case GGML_OP_ARGSORT:+ func = ggml_cuda_argsort;+ break;default:return false;}@@ -8223,7 +9234,9 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_int ggml_cuda_get_device_count() {int device_count;- CUDA_CHECK(cudaGetDeviceCount(&device_count));+ if (cudaGetDeviceCount(&device_count) != cudaSuccess) {+ return 0;+ }return device_count;}@@ -8232,3 +9245,601 @@ void ggml_cuda_get_device_description(int device, char * description, size_t desCUDA_CHECK(cudaGetDeviceProperties(&prop, device));snprintf(description, description_size, "%s", prop.name);}++////////////////////////////////////////////////////////////////////////////////++// backend interface++#define UNUSED GGML_UNUSED++// cuda buffer++struct ggml_backend_buffer_context_cuda {+ int device;+ void * dev_ptr = nullptr;+ ggml_tensor_extra_gpu * temp_tensor_extras = nullptr;+ size_t temp_tensor_extra_index = 0;++ ggml_backend_buffer_context_cuda(int device, void * dev_ptr) : device(device), dev_ptr(dev_ptr) {}++ ~ggml_backend_buffer_context_cuda() {+ delete[] temp_tensor_extras;+ }++ ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() {+ if (temp_tensor_extras == nullptr) {+ temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES];+ }++ size_t alloc_index = temp_tensor_extra_index;+ temp_tensor_extra_index = (temp_tensor_extra_index + 1) % GGML_CUDA_MAX_NODES;+ ggml_tensor_extra_gpu * extra = &temp_tensor_extras[alloc_index];+ memset(extra, 0, sizeof(*extra));++ return extra;+ }+};++static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ CUDA_CHECK(cudaFree(ctx->dev_ptr));+ delete ctx;+}++static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {+ ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ return ctx->dev_ptr;+}++static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {+ ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;++ if (tensor->view_src != NULL && tensor->view_offs == 0) {+ assert(tensor->view_src->buffer->buft == buffer->buft); // TODO+ tensor->backend = tensor->view_src->backend;+ tensor->extra = tensor->view_src->extra;+ return;+ }++ ggml_tensor_extra_gpu * extra = ctx->ggml_cuda_alloc_temp_tensor_extra();++ extra->data_device[ctx->device] = tensor->data;++ tensor->backend = GGML_BACKEND_GPU;+ tensor->extra = extra;++ if (ggml_is_quantized(tensor->type)) {+ // initialize padding to 0 to avoid possible NaN values+ int64_t row_low = 0;+ int64_t row_high = ggml_nrows(tensor);+ int64_t nrows_split = row_high - row_low;++ size_t original_size = ggml_nbytes_split(tensor, nrows_split);+ size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);++ if (padded_size > original_size && tensor->view_src == nullptr) {+ CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + original_size, 0, padded_size - original_size, g_cudaStreams[ctx->device][0]));+ }+ }++ UNUSED(buffer);+}++static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);++ CUDA_CHECK(cudaMemcpy((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice));++ UNUSED(buffer);+}++static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);++ CUDA_CHECK(cudaMemcpy(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost));++ UNUSED(buffer);+}++static struct ggml_backend_buffer_i cuda_backend_buffer_interface = {+ /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,+ /* .get_base = */ ggml_backend_cuda_buffer_get_base,+ /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor,+ /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,+ /* .cpy_tensor_from = */ NULL,+ /* .cpy_tensor_to = */ NULL,+};++// cuda buffer type++static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ int device = (int) (intptr_t) buft->context;++ ggml_cuda_set_device(device);++ size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0++ void * dev_ptr;+ CUDA_CHECK(cudaMalloc(&dev_ptr, size));++ ggml_backend_buffer_context_cuda * ctx = new ggml_backend_buffer_context_cuda(device, dev_ptr);++ return ggml_backend_buffer_init(buft, cuda_backend_buffer_interface, ctx, size);+}++static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+ return 128;++ UNUSED(buft);+}++static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, ggml_tensor * tensor) {+ int64_t row_low = 0;+ int64_t row_high = ggml_nrows(tensor);+ int64_t nrows_split = row_high - row_low;++ size_t size = ggml_nbytes_split(tensor, nrows_split);++ int64_t ne0 = tensor->ne[0];++ if (ggml_is_quantized(tensor->type)) {+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += (MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING)+ * ggml_type_size(tensor->type)/ggml_blck_size(tensor->type);+ }+ }++ return size;++ UNUSED(buft);+}++static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ return ggml_backend_is_cuda(backend);++ UNUSED(buft);+}++static ggml_backend_buffer_type_i cuda_backend_buffer_type_interface = {+ /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,+ /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,+ /* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend,+};++ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {+ static struct ggml_backend_buffer_type ggml_backend_buffer_type_cuda[GGML_CUDA_MAX_DEVICES];+ static bool ggml_backend_buffer_type_cuda_initialized = false;+ if (!ggml_backend_buffer_type_cuda_initialized) {+ for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) {+ ggml_backend_buffer_type_cuda[i] = {+ /* .iface = */ cuda_backend_buffer_type_interface,+ /* .context = */ (ggml_backend_buffer_type_context_t) (intptr_t) i,+ };+ }+ ggml_backend_buffer_type_cuda_initialized = true;+ }++ return &ggml_backend_buffer_type_cuda[device];+}++// host buffer type++static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ CUDA_CHECK(cudaFreeHost(ctx->dev_ptr));+ delete ctx;+}++static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ void * ptr;+ CUDA_CHECK(cudaMallocHost(&ptr, size));++ // FIXME: this is a hack to avoid having to implement a new buffer type+ ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);+ buffer->buft = buft;+ buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;++ return buffer;++ UNUSED(buft);+}++struct ggml_backend_buffer_type_i cuda_backend_host_buffer_type_interface = {+ /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,+ /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,+ /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend,+};++ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {+ static struct ggml_backend_buffer_type ggml_backend_buffer_type_cuda_host = {+ /* .iface = */ cuda_backend_host_buffer_type_interface,+ /* .context = */ nullptr,+ };++ return &ggml_backend_buffer_type_cuda_host;+}++// backend++struct ggml_backend_context_cuda {+ int device;+};++static const char * ggml_backend_cuda_name(ggml_backend_t backend) {+ return GGML_CUDA_NAME;++ UNUSED(backend);+}++static void ggml_backend_cuda_free(ggml_backend_t backend) {+ ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;++ delete cuda_ctx;+ delete backend;+}++static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer_type(ggml_backend_t backend) {+ ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;++ return ggml_backend_cuda_buffer_type(cuda_ctx->device);+}++static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;++ GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);++ CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, g_cudaStreams[cuda_ctx->device][0]));+}++static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;++ GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");+ GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);++ CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0]));+}++static void ggml_backend_cuda_synchronize(ggml_backend_t backend) {+ ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;++ CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[cuda_ctx->device][0]));++ UNUSED(backend);+}++static ggml_backend_graph_plan_t ggml_backend_cuda_graph_plan_create(ggml_backend_t backend, ggml_cgraph * cgraph) {+ GGML_ASSERT(!"not implemented");++ return nullptr;++ UNUSED(backend);+ UNUSED(cgraph);+}++static void ggml_backend_cuda_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {+ GGML_ASSERT(!"not implemented");++ UNUSED(backend);+ UNUSED(plan);+}++static void ggml_backend_cuda_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {+ GGML_ASSERT(!"not implemented");++ UNUSED(backend);+ UNUSED(plan);+}++static void ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {+ ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;++ ggml_cuda_set_main_device(cuda_ctx->device);++ ggml_compute_params params = {};+ params.type = GGML_TASK_COMPUTE;+ params.ith = 0;+ for (int i = 0; i < cgraph->n_nodes; i++) {+ ggml_tensor * node = cgraph->nodes[i];++ if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE)+ continue;++ assert(node->backend == GGML_BACKEND_GPU);+ assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));+ assert(node->extra != nullptr);++ for (int j = 0; j < GGML_MAX_SRC; j++) {+ if (node->src[j] != nullptr) {+ assert(node->src[j]->backend == GGML_BACKEND_GPU);+ assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));+ assert(node->src[j]->extra != nullptr);+ }+ }++ bool ok = ggml_cuda_compute_forward(¶ms, node);+ if (!ok) {+ fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));+ }+ GGML_ASSERT(ok);++#if 0+ if (node->type == GGML_TYPE_F32) {+ cudaDeviceSynchronize();+ std::vector<float> tmp(ggml_nelements(node), 0.0f);+ cudaMemcpy(tmp.data(), node->data, ggml_nelements(node)*sizeof(float), cudaMemcpyDeviceToHost);+ printf("\n%s (%s) (%s %s) (%s %s): ", node->name, ggml_op_name(node->op),+ ggml_type_name(node->src[0]->type),+ node->src[1] ? ggml_type_name(node->src[1]->type) : "none",+ node->src[0]->name,+ node->src[1] ? node->src[1]->name : "none");+ double sum = 0.0;+ double sq_sum = 0.0;+ for (int i = 0; i < ggml_nelements(node); i++) {+ printf("%f ", tmp[i]);+ sum += tmp[i];+ sq_sum += tmp[i]*tmp[i];+ }+ printf("\n");+ printf("sum: %f, ", sum);+ printf("sq_sum: %f\n", sq_sum);+ }+#endif+ }++ UNUSED(backend);+}++static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) {+ switch (op->op) {+ case GGML_OP_UNARY:+ switch (ggml_get_unary_op(op)) {+ case GGML_UNARY_OP_GELU:+ case GGML_UNARY_OP_SILU:+ case GGML_UNARY_OP_RELU:+ return true;+ default:+ return false;+ }+ break;+ case GGML_OP_MUL_MAT:+ case GGML_OP_MUL_MAT_ID:+ {+ struct ggml_tensor * a;+ struct ggml_tensor * b;+ if (op->op == GGML_OP_MUL_MAT) {+ a = op->src[0];+ b = op->src[1];+ } else {+ a = op->src[2];+ b = op->src[1];+ }+ if (a->ne[3] != b->ne[3]) {+ return false;+ }+ return true;+ } break;+ case GGML_OP_GET_ROWS:+ {+ switch (op->src[0]->type) {+ case GGML_TYPE_F16:+ case GGML_TYPE_F32:+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ return true;+ default:+ return false;+ }+ } break;+ case GGML_OP_CPY:+ {+ ggml_type src0_type = op->src[0]->type;+ ggml_type src1_type = op->src[1]->type;+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) {+ return true;+ }+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) {+ return true;+ }+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q8_0) {+ return true;+ }+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_0) {+ return true;+ }+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) {+ return true;+ }+ if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {+ return true;+ }+ return false;+ } break;+ case GGML_OP_NONE:+ case GGML_OP_RESHAPE:+ case GGML_OP_VIEW:+ case GGML_OP_PERMUTE:+ case GGML_OP_TRANSPOSE:+ case GGML_OP_NORM:+ case GGML_OP_REPEAT:+ case GGML_OP_DUP:+ case GGML_OP_ADD:+ case GGML_OP_MUL:+ case GGML_OP_DIV:+ case GGML_OP_RMS_NORM:+ case GGML_OP_SCALE:+ case GGML_OP_SQR:+ case GGML_OP_CLAMP:+ case GGML_OP_CONT:+ case GGML_OP_DIAG_MASK_INF:+ case GGML_OP_SOFT_MAX:+ case GGML_OP_ROPE:+ case GGML_OP_ALIBI:+ case GGML_OP_IM2COL:+ case GGML_OP_SUM_ROWS:+ case GGML_OP_ARGSORT:+ return true;+ default:+ return false;+ }++ UNUSED(backend);+}++static ggml_backend_i cuda_backend_i = {+ /* .get_name = */ ggml_backend_cuda_name,+ /* .free = */ ggml_backend_cuda_free,+ /* .get_default_buffer_type = */ ggml_backend_cuda_get_default_buffer_type,+ /* .set_tensor_async = */ ggml_backend_cuda_set_tensor_async,+ /* .get_tensor_async = */ ggml_backend_cuda_get_tensor_async,+ /* .cpy_tensor_from_async = */ NULL,+ /* .cpy_tensor_to_async = */ NULL,+ /* .synchronize = */ ggml_backend_cuda_synchronize,+ /* .graph_plan_create = */ ggml_backend_cuda_graph_plan_create,+ /* .graph_plan_free = */ ggml_backend_cuda_graph_plan_free,+ /* .graph_plan_compute = */ ggml_backend_cuda_graph_plan_compute,+ /* .graph_compute = */ ggml_backend_cuda_graph_compute,+ /* .supports_op = */ ggml_backend_cuda_supports_op,+};++ggml_backend_t ggml_backend_cuda_init(int device) {+ ggml_init_cublas(); // TODO: remove from ggml.c++ if (device < 0 || device >= ggml_cuda_get_device_count()) {+ fprintf(stderr, "%s: error: invalid device %d\n", __func__, device);+ return nullptr;+ }++ // not strictly necessary, but it may reduce the overhead of the first graph_compute+ ggml_cuda_set_main_device(device);++ ggml_backend_context_cuda * ctx = new ggml_backend_context_cuda {+ /* .device = */ device+ };++ ggml_backend_t cuda_backend = new ggml_backend {+ /* .interface = */ cuda_backend_i,+ /* .context = */ ctx+ };++ return cuda_backend;+}++bool ggml_backend_is_cuda(ggml_backend_t backend) {+ return backend->iface.get_name == ggml_backend_cuda_name;+}++ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {+ ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data);+ return cuda_backend;++ UNUSED(params);+}++extern "C" int ggml_backend_cuda_reg_devices();++static int ggml_blck_size(enum ggml_type type) {+ switch (type) {+ case GGML_TYPE_I8:+ return 1;+ case GGML_TYPE_I16:+ return 1;+ case GGML_TYPE_I32:+ return 1;+ case GGML_TYPE_F32:+ return 1;+ case GGML_TYPE_F16:+ return 1;+ case GGML_TYPE_Q4_0:+ return QK4_0;+ case GGML_TYPE_Q4_1:+ return QK4_1;+ case GGML_TYPE_Q5_0:+ return QK5_0;+ case GGML_TYPE_Q5_1:+ return QK5_1;+ case GGML_TYPE_Q8_0:+ return QK8_0;+ case GGML_TYPE_Q8_1:+ return QK8_1;+ case GGML_TYPE_Q2_K:+ return QK_K;+ case GGML_TYPE_Q3_K:+ return QK_K;+ case GGML_TYPE_Q4_K:+ return QK_K;+ case GGML_TYPE_Q5_K:+ return QK_K;+ case GGML_TYPE_Q6_K:+ return QK_K;+ case GGML_TYPE_Q8_K:+ return QK_K;+ default:+ return 0;+ }+}++// This is only used for intermediate quantization and dot products+typedef struct {+ float d; // delta+ int8_t qs[QK_K]; // quants+ int16_t bsums[QK_K/16]; // sum of quants in groups of 16+} block_q8_K;+static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");++static size_t ggml_type_size(enum ggml_type type) {+ switch (type) {+ case GGML_TYPE_I8:+ return sizeof(int8_t);+ case GGML_TYPE_I16:+ return sizeof(int16_t);+ case GGML_TYPE_I32:+ return sizeof(int32_t);+ case GGML_TYPE_F32:+ return sizeof(float);+ case GGML_TYPE_F16:+ return sizeof(ggml_fp16_t);+ case GGML_TYPE_Q4_0:+ return sizeof(block_q4_0);+ case GGML_TYPE_Q4_1:+ return sizeof(block_q4_1);+ case GGML_TYPE_Q5_0:+ return sizeof(block_q5_0);+ case GGML_TYPE_Q5_1:+ return sizeof(block_q5_1);+ case GGML_TYPE_Q8_0:+ return sizeof(block_q8_0);+ case GGML_TYPE_Q8_1:+ return sizeof(block_q8_1);+ case GGML_TYPE_Q2_K:+ return sizeof(block_q2_K);+ case GGML_TYPE_Q3_K:+ return sizeof(block_q3_K);+ case GGML_TYPE_Q4_K:+ return sizeof(block_q4_K);+ case GGML_TYPE_Q5_K:+ return sizeof(block_q5_K);+ case GGML_TYPE_Q6_K:+ return sizeof(block_q6_K);+ case GGML_TYPE_Q8_K:+ return sizeof(block_q8_K);+ default:+ return 0;+ }+}diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.hindex 0cb41da4..bc3ede6a 100644--- a/llama.cpp/ggml-cuda.h+++ b/llama.cpp/ggml-cuda.h@@ -1,6 +1,7 @@#pragma once#include "ggml.h"+#include "ggml-backend.h"#ifdef GGML_USE_HIPBLAS#define GGML_CUDA_NAME "ROCm"@@ -41,6 +42,7 @@ GGML_API void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, sizGGML_API void ggml_cuda_copy_to_device(struct ggml_tensor * tensor);GGML_API void ggml_cuda_set_main_device(int main_device);+GGML_API void ggml_cuda_set_mul_mat_q(bool mul_mat_q);GGML_API void ggml_cuda_set_scratch_size(size_t scratch_size);GGML_API void ggml_cuda_free_scratch(void);GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);@@ -48,6 +50,19 @@ GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, sGGML_API int ggml_cuda_get_device_count(void);GGML_API void ggml_cuda_get_device_description(int device, char * description, size_t description_size);+// backend API+GGML_API ggml_backend_t ggml_backend_cuda_init(int device);++GGML_API bool ggml_backend_is_cuda(ggml_backend_t backend);+GGML_API int ggml_backend_cuda_get_device(ggml_backend_t backend);++GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);++// pinned host buffer for use with CPU backend for faster copies between CPU and GPU+GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);++GGML_API ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data);+#ifdef __cplusplus}#endifdiff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex 298a0c89..d116adfe 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -39,12 +39,6 @@ extern "C" {#endif#endif-#undef MIN-#undef MAX--#define MIN(a, b) ((a) < (b) ? (a) : (b))-#define MAX(a, b) ((a) > (b) ? (a) : (b))-// 16-bit float// on Arm, we use __fp16// on x86, we use uint16_t@@ -54,7 +48,9 @@ extern "C" {//// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src///+#ifndef __CUDACC__#include <arm_neon.h>+#endif#define GGML_COMPUTE_FP16_TO_FP32(x) ((float) (x))#define GGML_COMPUTE_FP32_TO_FP16(x) (x)@@ -233,7 +229,19 @@ inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {#endif- // TODO: backend v2 PR+#define GGML_HASHTABLE_FULL ((size_t)-1)+#define GGML_HASHTABLE_ALREADY_EXISTS ((size_t)-2)++bool ggml_hash_contains (const struct ggml_hash_set hash_set, struct ggml_tensor * key);++// returns GGML_HASHTABLE_FULL if table is full, otherwise the current index of the key or where it should be inserted+size_t ggml_hash_find (const struct ggml_hash_set hash_set, struct ggml_tensor * key);++// returns GGML_HASHTABLE_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full+size_t ggml_hash_insert ( struct ggml_hash_set hash_set, struct ggml_tensor * key);++// return index, asserts if table is full+size_t ggml_hash_find_or_insert( struct ggml_hash_set hash_set, struct ggml_tensor * key);#ifdef __cplusplus}diff --git a/llama.cpp/ggml-metal.h b/llama.cpp/ggml-metal.hindex 5dd6bae9..0ff777d3 100644--- a/llama.cpp/ggml-metal.h+++ b/llama.cpp/ggml-metal.h@@ -20,12 +20,13 @@#pragma once#include "ggml.h"+#include "ggml-backend.h"#include <stddef.h>#include <stdbool.h>// max memory buffers that can be mapped to the device-#define GGML_METAL_MAX_BUFFERS 16+#define GGML_METAL_MAX_BUFFERS 64#define GGML_METAL_MAX_COMMAND_BUFFERS 32struct ggml_tensor;@@ -45,7 +46,7 @@ struct ggml_metal_context;bool ggml_metal_supported(void);// number of command buffers to use-struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath);+struct ggml_metal_context * ggml_metal_init(int n_cb);void ggml_metal_free(struct ggml_metal_context * ctx);void * ggml_metal_host_malloc(size_t n);@@ -88,6 +89,23 @@ int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx);// creates gf->n_threads command buffers in parallelvoid ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);+//+// backend API+// user-code should use only these functions+//++GGML_API ggml_backend_t ggml_backend_metal_init(void);++GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);++GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);+GGML_API ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);++// helper to check if the device supports a specific family+// ideally, the user code should be doing these checks+// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf+GGML_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);+#ifdef __cplusplus}#endifdiff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex 9b46f781..4eaa0198 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -1,9 +1,59 @@// -*- mode:objc;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+// vi: set et ft=objc ts=4 sts=4 sw=4 fenc=utf-8 :vi+#import "ggml-metal.h"+#import "ggml-quants.h"++#import "ggml-backend-impl.h"+#import "ggml.h"+#import <Foundation/Foundation.h>+#import <Metal/Metal.h>-#include "ggml-metal.h"-#include "ggml-quants.h"++////////////////////////////////////////////////////////////////////////////////+// BEGIN: COPIED FROM GGML-BACKEND.C++#define ggml_backend_buffer_init ggml_backend_buffer_init_+static ggml_backend_buffer_t ggml_backend_buffer_init(+ ggml_backend_buffer_type_t buft,+ struct ggml_backend_buffer_i iface,+ ggml_backend_buffer_context_t context,+ size_t size) {+ // TODO(jart): this malloc() looks problematic+ // what does this backend framework do?+ fprintf(stderr, "WARNING: using untested foreign malloc due to ggml backend\n");+ ggml_backend_buffer_t buffer = malloc(sizeof(struct ggml_backend_buffer));+ (*buffer) = (struct ggml_backend_buffer) {+ /* .interface = */ iface,+ /* .buft = */ buft,+ /* .context = */ context,+ /* .size = */ size,+ };+ return buffer;+}++static const char * ggml_backend_cpu_name(ggml_backend_t backend) {+ return "CPU";+ GGML_UNUSED(backend);+}++#define ggml_backend_is_cpu ggml_backend_is_cpu_+static bool ggml_backend_is_cpu(ggml_backend_t backend) {+ return backend->iface.get_name == ggml_backend_cpu_name;+}++#define ggml_backend_tensor_get ggml_backend_tensor_get_+static void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ tensor->buffer->iface.get_tensor(tensor->buffer, tensor, data, offset, size);+}++#define ggml_backend_tensor_set ggml_backend_tensor_set_+static void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ tensor->buffer->iface.set_tensor(tensor->buffer, tensor, data, offset, size);+}++// END: COPIED FROM GGML-BACKEND.C+////////////////////////////////////////////////////////////////////////////////#undef MIN#undef MAX@@ -15,56 +65,57 @@#define GGML_METAL_LOG_WARN(...)#define GGML_METAL_LOG_ERROR(...)#else-#define GGML_METAL_LOG_INFO(...) fprintf(stderr, __VA_ARGS__)-#define GGML_METAL_LOG_WARN(...) fprintf(stderr, __VA_ARGS__)-#define GGML_METAL_LOG_ERROR(...) fprintf(stderr, __VA_ARGS__)+#define GGML_METAL_LOG_INFO(...) ggml_metal_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)+#define GGML_METAL_LOG_WARN(...) ggml_metal_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)+#define GGML_METAL_LOG_ERROR(...) ggml_metal_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)#endif-#define GGML_METAL_MAX_BUFFERS 16-#define GGML_METAL_MAX_COMMAND_BUFFERS 32+#define UNUSED(x) (void)(x)++#define GGML_MAX_CONCUR (2*GGML_DEFAULT_GRAPH_SIZE)////////////////////////////////////////////////////////////////////////////////// BEGIN: COPIED FROM GGML.C-#define ggml_blck_size ggml_blck_size_-static int ggml_blck_size(enum ggml_type type) {+#define ggml_is_quantized ggml_is_quantized_+static bool ggml_is_quantized(enum ggml_type type) {switch (type) {case GGML_TYPE_I8:- return 1;+ return false;case GGML_TYPE_I16:- return 1;+ return false;case GGML_TYPE_I32:- return 1;+ return false;case GGML_TYPE_F32:- return 1;+ return false;case GGML_TYPE_F16:- return 1;+ return false;case GGML_TYPE_Q4_0:- return QK4_0;+ return true;case GGML_TYPE_Q4_1:- return QK4_1;+ return true;case GGML_TYPE_Q5_0:- return QK5_0;+ return true;case GGML_TYPE_Q5_1:- return QK5_1;+ return true;case GGML_TYPE_Q8_0:- return QK8_0;+ return true;case GGML_TYPE_Q8_1:- return QK8_1;+ return true;case GGML_TYPE_Q2_K:- return QK_K;+ return true;case GGML_TYPE_Q3_K:- return QK_K;+ return true;case GGML_TYPE_Q4_K:- return QK_K;+ return true;case GGML_TYPE_Q5_K:- return QK_K;+ return true;case GGML_TYPE_Q6_K:- return QK_K;+ return true;case GGML_TYPE_Q8_K:- return QK_K;+ return true;default:- return 0;+ return false;}}@@ -110,6 +161,53 @@ static size_t ggml_type_size(enum ggml_type type) {}}+#define ggml_blck_size ggml_blck_size_+static int ggml_blck_size(enum ggml_type type) {+ switch (type) {+ case GGML_TYPE_I8:+ return 1;+ case GGML_TYPE_I16:+ return 1;+ case GGML_TYPE_I32:+ return 1;+ case GGML_TYPE_F32:+ return 1;+ case GGML_TYPE_F16:+ return 1;+ case GGML_TYPE_Q4_0:+ return QK4_0;+ case GGML_TYPE_Q4_1:+ return QK4_1;+ case GGML_TYPE_Q5_0:+ return QK5_0;+ case GGML_TYPE_Q5_1:+ return QK5_1;+ case GGML_TYPE_Q8_0:+ return QK8_0;+ case GGML_TYPE_Q8_1:+ return QK8_1;+ case GGML_TYPE_Q2_K:+ return QK_K;+ case GGML_TYPE_Q3_K:+ return QK_K;+ case GGML_TYPE_Q4_K:+ return QK_K;+ case GGML_TYPE_Q5_K:+ return QK_K;+ case GGML_TYPE_Q6_K:+ return QK_K;+ case GGML_TYPE_Q8_K:+ return QK_K;+ default:+ return 0;+ }+}++#define ggml_is_transposed ggml_is_transposed_+static bool ggml_is_transposed(const struct ggml_tensor * tensor) {+ return tensor->nb[0] > tensor->nb[1];+}+#define ggml_nbytes ggml_nbytes_static size_t ggml_nbytes(const struct ggml_tensor * tensor) {size_t nbytes;@@ -129,24 +227,13 @@ static size_t ggml_nbytes(const struct ggml_tensor * tensor) {return nbytes;}-#define ggml_is_transposed ggml_is_transposed_-static bool ggml_is_transposed(const struct ggml_tensor * tensor) {- return tensor->nb[0] > tensor->nb[1];-}--#define ggml_is_contiguous ggml_is_contiguous_-static bool ggml_is_contiguous(const struct ggml_tensor * tensor) {- static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");- return- tensor->nb[0] == ggml_type_size(tensor->type) &&- tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&- tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&- tensor->nb[3] == tensor->nb[2]*tensor->ne[2];+static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {+ return ((const int32_t *)(tensor->op_params))[i];}-#define ggml_nrows ggml_nrows_-static int64_t ggml_nrows(const struct ggml_tensor * tensor) {- return tensor->ne[1]*tensor->ne[2]*tensor->ne[3];+#define ggml_get_unary_op ggml_get_unary_op_+static enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) {+ return (enum ggml_unary_op) ggml_get_op_params_i32(tensor, 0);}#define ggml_nelements ggml_nelements_@@ -154,22 +241,28 @@ static int64_t ggml_nelements(const struct ggml_tensor * tensor) {return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3];}-static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {- return ((const int32_t *)(tensor->op_params))[i];+#define ggml_nrows ggml_nrows_+static int64_t ggml_nrows(const struct ggml_tensor * tensor) {+ return tensor->ne[1]*tensor->ne[2]*tensor->ne[3];}-#define ggml_get_unary_op ggml_get_unary_op_-static enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) {- return (enum ggml_unary_op) ggml_get_op_params_i32(tensor, 0);+#define ggml_is_contiguous ggml_is_contiguous_+static bool ggml_is_contiguous(const struct ggml_tensor * tensor) {+ return+ tensor->nb[0] == ggml_type_size(tensor->type) &&+ tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&+ tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&+ tensor->nb[3] == tensor->nb[2]*tensor->ne[2];+}++#define ggml_op_name ggml_op_name_+static const char * ggml_op_name(enum ggml_op op) {+ return "REDACTED!GGML_OP_NAME[op]";}// END: COPIED FROM GGML.C////////////////////////////////////////////////////////////////////////////////-#define UNUSED(x) (void)(x)--#define GGML_MAX_CONCUR (2*GGML_MAX_NODES)-struct ggml_metal_buffer {const char * name;@@ -206,6 +299,8 @@ struct ggml_metal_context {GGML_METAL_DECL_KERNEL(add_row); // TODO: avoid this extra kernel, instead extend the "add" kernel to support broadcastGGML_METAL_DECL_KERNEL(mul);GGML_METAL_DECL_KERNEL(mul_row); // TODO: avoid this extra kernel, instead extend the "mul" kernel to support broadcast+ GGML_METAL_DECL_KERNEL(div);+ GGML_METAL_DECL_KERNEL(div_row);GGML_METAL_DECL_KERNEL(scale);GGML_METAL_DECL_KERNEL(scale_4);GGML_METAL_DECL_KERNEL(silu);@@ -230,6 +325,7 @@ struct ggml_metal_context {GGML_METAL_DECL_KERNEL(rms_norm);GGML_METAL_DECL_KERNEL(norm);GGML_METAL_DECL_KERNEL(mul_mv_f32_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_f16_f16);GGML_METAL_DECL_KERNEL(mul_mv_f16_f32);GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_1row);GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_l4);@@ -243,6 +339,21 @@ struct ggml_metal_context {GGML_METAL_DECL_KERNEL(mul_mv_q4_K_f32);GGML_METAL_DECL_KERNEL(mul_mv_q5_K_f32);GGML_METAL_DECL_KERNEL(mul_mv_q6_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_f32_f32);+ //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f16);+ GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32);+ //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_1row);+ //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_l4);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q4_0_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q4_1_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q5_0_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q5_1_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q8_0_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q2_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q3_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q4_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q5_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mv_id_q6_K_f32);GGML_METAL_DECL_KERNEL(mul_mm_f32_f32);GGML_METAL_DECL_KERNEL(mul_mm_f16_f32);GGML_METAL_DECL_KERNEL(mul_mm_q4_0_f32);@@ -255,14 +366,36 @@ struct ggml_metal_context {GGML_METAL_DECL_KERNEL(mul_mm_q4_K_f32);GGML_METAL_DECL_KERNEL(mul_mm_q5_K_f32);GGML_METAL_DECL_KERNEL(mul_mm_q6_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_f32_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_f16_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q4_0_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q4_1_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q5_0_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q5_1_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q8_0_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q2_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q3_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q4_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q5_K_f32);+ GGML_METAL_DECL_KERNEL(mul_mm_id_q6_K_f32);GGML_METAL_DECL_KERNEL(rope_f32);GGML_METAL_DECL_KERNEL(rope_f16);GGML_METAL_DECL_KERNEL(alibi_f32);+ GGML_METAL_DECL_KERNEL(im2col_f16);+ GGML_METAL_DECL_KERNEL(argsort_f32_i32_asc);+ GGML_METAL_DECL_KERNEL(argsort_f32_i32_desc);GGML_METAL_DECL_KERNEL(cpy_f32_f16);GGML_METAL_DECL_KERNEL(cpy_f32_f32);+ GGML_METAL_DECL_KERNEL(cpy_f32_q8_0);+ GGML_METAL_DECL_KERNEL(cpy_f32_q4_0);+ GGML_METAL_DECL_KERNEL(cpy_f32_q4_1);+ //GGML_METAL_DECL_KERNEL(cpy_f32_q5_0);+ //GGML_METAL_DECL_KERNEL(cpy_f32_q5_1);GGML_METAL_DECL_KERNEL(cpy_f16_f16);+ GGML_METAL_DECL_KERNEL(cpy_f16_f32);GGML_METAL_DECL_KERNEL(concat);GGML_METAL_DECL_KERNEL(sqr);+ GGML_METAL_DECL_KERNEL(sum_rows);#undef GGML_METAL_DECL_KERNEL};@@ -270,7 +403,7 @@ struct ggml_metal_context {// MSL code// TODO: move the contents here when ready// for now it is easier to work in a separate file-static NSString * const msl_library_source = @"see metal.metal";+//static NSString * const msl_library_source = @"see metal.metal";// Here to assist with NSBundle Path Hack@interface GGMLMetalClass : NSObject@@ -278,10 +411,40 @@ static NSString * const msl_library_source = @"see metal.metal";@implementation GGMLMetalClass@end-struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath) {+ggml_log_callback ggml_metal_log_callback = NULL;+void * ggml_metal_log_user_data = NULL;++static void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {+ ggml_metal_log_callback = log_callback;+ ggml_metal_log_user_data = user_data;+}++GGML_ATTRIBUTE_FORMAT(2, 3)+static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){+ if (ggml_metal_log_callback != NULL) {+ va_list args;+ va_start(args, format);+ char buffer[128];+ int len = vsnprintf(buffer, 128, format, args);+ if (len < 128) {+ ggml_metal_log_callback(level, buffer, ggml_metal_log_user_data);+ } else {+ char* buffer2 = malloc(len+1);+ va_end(args);+ va_start(args, format);+ vsnprintf(buffer2, len+1, format, args);+ buffer2[len] = 0;+ ggml_metal_log_callback(level, buffer2, ggml_metal_log_user_data);+ free(buffer2);+ }+ va_end(args);+ }+}++struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_LOG_INFO("%s: allocating\n", __func__);- id <MTLDevice> device;+ id<MTLDevice> device;NSString * s;#if TARGET_OS_OSX@@ -325,9 +488,19 @@ struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath) {} else {GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);- NSString * sourcePath = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];+ NSString * sourcePath;+ NSString * ggmlMetalPathResources = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];++ GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, ggmlMetalPathResources ? [ggmlMetalPathResources UTF8String] : "nil");++ if (ggmlMetalPathResources) {+ sourcePath = [ggmlMetalPathResources stringByAppendingPathComponent:@"ggml-metal.metal"];+ } else {+ sourcePath = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];+ }if (sourcePath == nil) {- sourcePath = [NSString stringWithUTF8String:metalPath];+ GGML_METAL_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__);+ sourcePath = @"ggml-metal.metal";}GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [sourcePath UTF8String]);NSString * src = [NSString stringWithContentsOfFile:sourcePath encoding:NSUTF8StringEncoding error:&error];@@ -350,6 +523,29 @@ struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath) {}}+#if TARGET_OS_OSX+ // print MTL GPU family:+ GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]);++ // determine max supported GPU family+ // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf+ // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {+ if ([ctx->device supportsFamily:i]) {+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);+ break;+ }+ }++ GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");+ GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);+ if (ctx->device.maxTransferRate != 0) {+ GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1e6);+ } else {+ GGML_METAL_LOG_INFO("%s: maxTransferRate = built-in GPU\n", __func__);+ }+#endif+// load kernels{NSError * error = nil;@@ -371,6 +567,8 @@ struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath) {GGML_METAL_ADD_KERNEL(add_row);GGML_METAL_ADD_KERNEL(mul);GGML_METAL_ADD_KERNEL(mul_row);+ GGML_METAL_ADD_KERNEL(div);+ GGML_METAL_ADD_KERNEL(div_row);GGML_METAL_ADD_KERNEL(scale);GGML_METAL_ADD_KERNEL(scale_4);GGML_METAL_ADD_KERNEL(silu);@@ -395,6 +593,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath) {GGML_METAL_ADD_KERNEL(rms_norm);GGML_METAL_ADD_KERNEL(norm);GGML_METAL_ADD_KERNEL(mul_mv_f32_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_f16_f16);GGML_METAL_ADD_KERNEL(mul_mv_f16_f32);GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_1row);GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_l4);@@ -408,6 +607,21 @@ struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath) {GGML_METAL_ADD_KERNEL(mul_mv_q4_K_f32);GGML_METAL_ADD_KERNEL(mul_mv_q5_K_f32);GGML_METAL_ADD_KERNEL(mul_mv_q6_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_f32_f32);+ //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f16);+ GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32);+ //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_1row);+ //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_l4);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q4_0_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q4_1_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q5_0_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q5_1_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q8_0_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q2_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q3_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q4_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q5_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mv_id_q6_K_f32);if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {GGML_METAL_ADD_KERNEL(mul_mm_f32_f32);GGML_METAL_ADD_KERNEL(mul_mm_f16_f32);@@ -421,42 +635,41 @@ struct ggml_metal_context * ggml_metal_init(int n_cb, const char *metalPath) {GGML_METAL_ADD_KERNEL(mul_mm_q4_K_f32);GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32);GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_f32_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_f16_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q4_0_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q4_1_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q5_0_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q5_1_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q8_0_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q2_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q3_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q4_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q5_K_f32);+ GGML_METAL_ADD_KERNEL(mul_mm_id_q6_K_f32);}GGML_METAL_ADD_KERNEL(rope_f32);GGML_METAL_ADD_KERNEL(rope_f16);GGML_METAL_ADD_KERNEL(alibi_f32);+ GGML_METAL_ADD_KERNEL(im2col_f16);+ GGML_METAL_ADD_KERNEL(argsort_f32_i32_asc);+ GGML_METAL_ADD_KERNEL(argsort_f32_i32_desc);GGML_METAL_ADD_KERNEL(cpy_f32_f16);GGML_METAL_ADD_KERNEL(cpy_f32_f32);+ GGML_METAL_ADD_KERNEL(cpy_f32_q8_0);+ GGML_METAL_ADD_KERNEL(cpy_f32_q4_0);+ GGML_METAL_ADD_KERNEL(cpy_f32_q4_1);+ //GGML_METAL_ADD_KERNEL(cpy_f32_q5_0);+ //GGML_METAL_ADD_KERNEL(cpy_f32_q5_1);GGML_METAL_ADD_KERNEL(cpy_f16_f16);+ GGML_METAL_ADD_KERNEL(cpy_f16_f32);GGML_METAL_ADD_KERNEL(concat);GGML_METAL_ADD_KERNEL(sqr);+ GGML_METAL_ADD_KERNEL(sum_rows);#undef GGML_METAL_ADD_KERNEL}-#if TARGET_OS_OSX- // print MTL GPU family:- GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]);-- // determine max supported GPU family- // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf- // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf- for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {- if ([ctx->device supportsFamily:i]) {- GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%ld (%d)\n", __func__, i - MTLGPUFamilyApple1 + 1, i);- break;- }- }-- GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");- GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);- if (ctx->device.maxTransferRate != 0) {- GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1024.0 / 1024.0);- } else {- GGML_METAL_LOG_INFO("%s: maxTransferRate = built-in GPU\n", __func__);- }-#endif-return ctx;}@@ -470,6 +683,8 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {GGML_METAL_DEL_KERNEL(add_row);GGML_METAL_DEL_KERNEL(mul);GGML_METAL_DEL_KERNEL(mul_row);+ GGML_METAL_DEL_KERNEL(div);+ GGML_METAL_DEL_KERNEL(div_row);GGML_METAL_DEL_KERNEL(scale);GGML_METAL_DEL_KERNEL(scale_4);GGML_METAL_DEL_KERNEL(silu);@@ -494,6 +709,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {GGML_METAL_DEL_KERNEL(rms_norm);GGML_METAL_DEL_KERNEL(norm);GGML_METAL_DEL_KERNEL(mul_mv_f32_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_f16_f16);GGML_METAL_DEL_KERNEL(mul_mv_f16_f32);GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_1row);GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_l4);@@ -507,6 +723,21 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {GGML_METAL_DEL_KERNEL(mul_mv_q4_K_f32);GGML_METAL_DEL_KERNEL(mul_mv_q5_K_f32);GGML_METAL_DEL_KERNEL(mul_mv_q6_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_f32_f32);+ //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f16);+ GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32);+ //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_1row);+ //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_l4);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q4_0_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q4_1_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q5_0_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q5_1_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q8_0_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q2_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q3_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q4_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q5_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mv_id_q6_K_f32);if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {GGML_METAL_DEL_KERNEL(mul_mm_f32_f32);GGML_METAL_DEL_KERNEL(mul_mm_f16_f32);@@ -520,15 +751,37 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {GGML_METAL_DEL_KERNEL(mul_mm_q4_K_f32);GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32);GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_f32_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_f16_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q4_0_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q4_1_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q5_0_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q5_1_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q8_0_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q2_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q3_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q4_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q5_K_f32);+ GGML_METAL_DEL_KERNEL(mul_mm_id_q6_K_f32);}GGML_METAL_DEL_KERNEL(rope_f32);GGML_METAL_DEL_KERNEL(rope_f16);GGML_METAL_DEL_KERNEL(alibi_f32);+ GGML_METAL_DEL_KERNEL(im2col_f16);+ GGML_METAL_DEL_KERNEL(argsort_f32_i32_asc);+ GGML_METAL_DEL_KERNEL(argsort_f32_i32_desc);GGML_METAL_DEL_KERNEL(cpy_f32_f16);GGML_METAL_DEL_KERNEL(cpy_f32_f32);+ GGML_METAL_DEL_KERNEL(cpy_f32_q8_0);+ GGML_METAL_DEL_KERNEL(cpy_f32_q4_0);+ GGML_METAL_DEL_KERNEL(cpy_f32_q4_1);+ //GGML_METAL_DEL_KERNEL(cpy_f32_q5_0);+ //GGML_METAL_DEL_KERNEL(cpy_f32_q5_1);GGML_METAL_DEL_KERNEL(cpy_f16_f16);+ GGML_METAL_DEL_KERNEL(cpy_f16_f32);GGML_METAL_DEL_KERNEL(concat);GGML_METAL_DEL_KERNEL(sqr);+ GGML_METAL_DEL_KERNEL(sum_rows);#undef GGML_METAL_DEL_KERNEL@@ -572,6 +825,13 @@ int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx) {return ctx->concur_list;}+// temporarily defined here for compatibility between ggml-backend and the old API+struct ggml_backend_metal_buffer_context {+ void * data;++ id<MTLBuffer> metal;+};+// finds the Metal buffer that contains the tensor data on the GPU device// the assumption is that there is 1-to-1 mapping between the host and device memory buffers, so we can find the// Metal buffer based on the host memory pointer@@ -581,6 +841,19 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_metal_context * ctx, struconst int64_t tsize = ggml_nbytes(t);+ // compatibility with ggml-backend+ if (t->buffer && t->buffer->buft == ggml_backend_metal_buffer_type()) {+ struct ggml_backend_metal_buffer_context * buf_ctx = (struct ggml_backend_metal_buffer_context *) t->buffer->context;++ const int64_t ioffs = (int64_t) t->data - (int64_t) buf_ctx->data;++ GGML_ASSERT(ioffs >= 0 && ioffs + tsize <= (int64_t) t->buffer->size);++ *offs = (size_t) ioffs;++ return buf_ctx->metal;+ }+// find the view that contains the tensor fullyfor (int i = 0; i < ctx->n_buffers; ++i) {const int64_t ioffs = (int64_t) t->data - (int64_t) ctx->buffers[i].data;@@ -638,11 +911,11 @@ bool ggml_metal_add_buffer(ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];if (ctx->buffers[ctx->n_buffers].metal == nil) {- GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MB\n", __func__, name, size_aligned / 1024.0 / 1024.0);+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_aligned / 1024.0 / 1024.0);return false;}- GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MB", __func__, name, size_aligned / 1024.0 / 1024.0);+ GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB", __func__, name, size_aligned / 1024.0 / 1024.0);++ctx->n_buffers;} else {@@ -662,11 +935,11 @@ bool ggml_metal_add_buffer(ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];if (ctx->buffers[ctx->n_buffers].metal == nil) {- GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MB\n", __func__, name, size_step_aligned / 1024.0 / 1024.0);+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_step_aligned / 1024.0 / 1024.0);return false;}- GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MB, offs = %12ld", __func__, name, size_step_aligned / 1024.0 / 1024.0, i);+ GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB, offs = %12ld", __func__, name, size_step_aligned / 1024.0 / 1024.0, i);if (i + size_step < size) {GGML_METAL_LOG_INFO("\n");}@@ -681,7 +954,7 @@ bool ggml_metal_add_buffer(ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);if (ctx->device.currentAllocatedSize > ctx->device.recommendedMaxWorkingSetSize) {- GGML_METAL_LOG_WARN(", warning: current allocated size is greater than the recommended max working set size\n");+ GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);} else {GGML_METAL_LOG_INFO("\n");}@@ -803,6 +1076,76 @@ void ggml_metal_graph_find_concurrency(}}+static bool ggml_metal_supports_op(const struct ggml_tensor * op) {+ switch (op->op) {+ case GGML_OP_UNARY:+ switch (ggml_get_unary_op(op)) {+ case GGML_UNARY_OP_SILU:+ case GGML_UNARY_OP_RELU:+ case GGML_UNARY_OP_GELU:+ return true;+ default:+ return false;+ }+ case GGML_OP_NONE:+ case GGML_OP_RESHAPE:+ case GGML_OP_VIEW:+ case GGML_OP_PERMUTE:+ case GGML_OP_TRANSPOSE:+ case GGML_OP_GET_ROWS:+ case GGML_OP_CONCAT:+ case GGML_OP_ADD:+ case GGML_OP_MUL:+ case GGML_OP_DIV:+ case GGML_OP_SCALE:+ case GGML_OP_SQR:+ case GGML_OP_SUM_ROWS:+ case GGML_OP_SOFT_MAX:+ case GGML_OP_RMS_NORM:+ case GGML_OP_NORM:+ case GGML_OP_ALIBI:+ case GGML_OP_ROPE:+ case GGML_OP_IM2COL:+ case GGML_OP_ARGSORT:+ case GGML_OP_MUL_MAT:+ case GGML_OP_MUL_MAT_ID:+ return true;+ case GGML_OP_CPY:+ case GGML_OP_DUP:+ case GGML_OP_CONT:+ {+ switch (op->src[0]->type) {+ case GGML_TYPE_F32:+ switch (op->type) {+ case GGML_TYPE_F16:+ case GGML_TYPE_F32:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ return true;+ default:+ return false;+ }+ case GGML_TYPE_F16:+ switch (op->type) {+ case GGML_TYPE_F16:+ case GGML_TYPE_F32:+ return true;+ default:+ return false;+ }+ default:+ return false;+ };+ }+ case GGML_OP_DIAG_MASK_INF:+ {+ return op->ne[0] % 4 == 0;+ }+ default:+ return false;+ }+}void ggml_metal_graph_compute(struct ggml_metal_context * ctx,struct ggml_cgraph * gf) {@@ -859,6 +1202,22 @@ void ggml_metal_graph_compute(struct ggml_tensor * src1 = gf->nodes[i]->src[1];struct ggml_tensor * dst = gf->nodes[i];+ switch (dst->op) {+ case GGML_OP_NONE:+ case GGML_OP_RESHAPE:+ case GGML_OP_VIEW:+ case GGML_OP_TRANSPOSE:+ case GGML_OP_PERMUTE:+ {+ // noop -> next node+ } continue;+ default:+ {+ } break;+ }++ GGML_ASSERT(ggml_metal_supports_op(dst));+const int64_t ne00 = src0 ? src0->ne[0] : 0;const int64_t ne01 = src0 ? src0->ne[1] : 0;const int64_t ne02 = src0 ? src0->ne[2] : 0;@@ -912,14 +1271,6 @@ void ggml_metal_graph_compute(//}switch (dst->op) {- case GGML_OP_NONE:- case GGML_OP_RESHAPE:- case GGML_OP_VIEW:- case GGML_OP_TRANSPOSE:- case GGML_OP_PERMUTE:- {- // noop- } break;case GGML_OP_CONCAT:{const int64_t nb = ne00;@@ -959,25 +1310,40 @@ void ggml_metal_graph_compute([encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];} break;case GGML_OP_ADD:+ case GGML_OP_MUL:+ case GGML_OP_DIV:{- GGML_ASSERT(ggml_is_contiguous(src0));- GGML_ASSERT(ggml_is_contiguous(src1));-bool bcast_row = false;int64_t nb = ne00;- if (ggml_nelements(src1) == ne10 && ne00 % 4 == 0) {+ id<MTLComputePipelineState> pipeline = nil;++ if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {+ GGML_ASSERT(ggml_is_contiguous(src0));+// src1 is a rowGGML_ASSERT(ne11 == 1);nb = ne00 / 4;- [encoder setComputePipelineState:ctx->pipeline_add_row];+ switch (dst->op) {+ case GGML_OP_ADD: pipeline = ctx->pipeline_add_row; break;+ case GGML_OP_MUL: pipeline = ctx->pipeline_mul_row; break;+ case GGML_OP_DIV: pipeline = ctx->pipeline_div_row; break;+ default: GGML_ASSERT(false);+ }bcast_row = true;} else {- [encoder setComputePipelineState:ctx->pipeline_add];+ switch (dst->op) {+ case GGML_OP_ADD: pipeline = ctx->pipeline_add; break;+ case GGML_OP_MUL: pipeline = ctx->pipeline_mul; break;+ case GGML_OP_DIV: pipeline = ctx->pipeline_div; break;+ default: GGML_ASSERT(false);+ }}++ [encoder setComputePipelineState:pipeline];[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];[encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];[encoder setBuffer:id_dst offset:offs_dst atIndex:2];@@ -1012,36 +1378,11 @@ void ggml_metal_graph_compute([encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];} else {- const int nth = MIN(1024, ne0);+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);[encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];}} break;- case GGML_OP_MUL:- {- GGML_ASSERT(ggml_is_contiguous(src0));- GGML_ASSERT(ggml_is_contiguous(src1));-- // utilize float4- GGML_ASSERT(ne00 % 4 == 0);- const int64_t nb = ne00/4;-- if (ggml_nelements(src1) == ne10) {- // src1 is a row- GGML_ASSERT(ne11 == 1);- [encoder setComputePipelineState:ctx->pipeline_mul_row];- } else {- [encoder setComputePipelineState:ctx->pipeline_mul];- }- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&nb length:sizeof(nb) atIndex:3];-- const int64_t n = ggml_nelements(dst)/4;-- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;case GGML_OP_SCALE:{GGML_ASSERT(ggml_is_contiguous(src0));@@ -1099,7 +1440,7 @@ void ggml_metal_graph_compute(} break;default:{- GGML_METAL_LOG_WARN("%s: node %3d, op = %d not implemented\n", __func__, i, dst->op);+ GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));GGML_ASSERT(false);}} break;@@ -1114,33 +1455,78 @@ void ggml_metal_graph_compute(const int64_t n = ggml_nelements(dst);[encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];} break;- case GGML_OP_SOFT_MAX:+ case GGML_OP_SUM_ROWS:{- int nth = 32; // SIMD width+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));- if (ne00%4 == 0) {- [encoder setComputePipelineState:ctx->pipeline_soft_max_4];- } else {- do {- nth *= 2;- } while (nth <= ne00 && nth <= 1024);- nth /= 2;- [encoder setComputePipelineState:ctx->pipeline_soft_max];- }+ [encoder setComputePipelineState:ctx->pipeline_sum_rows];[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];[encoder setBuffer:id_dst offset:offs_dst atIndex:1];[encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];[encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];[encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];- [encoder setThreadgroupMemoryLength:MAX(16, nth/32*sizeof(float)) atIndex:0];-- [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19];+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20];+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21];+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23];+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24];+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25];++ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];} break;- case GGML_OP_DIAG_MASK_INF:+ case GGML_OP_SOFT_MAX:{- const int n_past = ((int32_t *)(dst->op_params))[0];+ int nth = 32; // SIMD width- if (ne00%8 == 0) {+ if (ne00%4 == 0) {+ while (nth < ne00/4 && nth < 256) {+ nth *= 2;+ }+ [encoder setComputePipelineState:ctx->pipeline_soft_max_4];+ } else {+ while (nth < ne00 && nth < 1024) {+ nth *= 2;+ }+ [encoder setComputePipelineState:ctx->pipeline_soft_max];+ }++ const float scale = ((float *) dst->op_params)[0];++ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ if (id_src1) {+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ } else {+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];+ }+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];+ [encoder setBytes:&scale length:sizeof(scale) atIndex:6];+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];++ [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_DIAG_MASK_INF:+ {+ const int n_past = ((int32_t *)(dst->op_params))[0];++ if (ne00%8 == 0) {[encoder setComputePipelineState:ctx->pipeline_diag_mask_inf_8];} else {[encoder setComputePipelineState:ctx->pipeline_diag_mask_inf];@@ -1161,9 +1547,13 @@ void ggml_metal_graph_compute(case GGML_OP_MUL_MAT:{GGML_ASSERT(ne00 == ne10);- GGML_ASSERT(ne03 == ne13);- const uint gqa = ne12/ne02;+ // TODO: assert that dim2 and dim3 are contiguous+ GGML_ASSERT(ne12 % ne02 == 0);+ GGML_ASSERT(ne13 % ne03 == 0);++ const uint r2 = ne12/ne02;+ const uint r3 = ne13/ne03;// find the break-even point where the matrix-matrix kernel becomes more efficient compared// to the matrix-vector kernel@@ -1198,7 +1588,7 @@ void ggml_metal_graph_compute(!ggml_is_transposed(src1) &&src1t == GGML_TYPE_F32 &&ne00 % 32 == 0 && ne00 >= 64 &&- ne11 > ne11_mm_min) {+ (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {//printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);switch (src0->type) {case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f32_f32]; break;@@ -1228,9 +1618,10 @@ void ggml_metal_graph_compute([encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];[encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];[encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];- [encoder setBytes:&gqa length:sizeof(gqa) atIndex:13];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:13];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:14];[encoder setThreadgroupMemoryLength:8192 atIndex:0];- [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];} else {int nth0 = 32;int nth1 = 1;@@ -1241,6 +1632,7 @@ void ggml_metal_graph_compute(switch (src0t) {case GGML_TYPE_F32:{+ GGML_ASSERT(src1t == GGML_TYPE_F32);[encoder setComputePipelineState:ctx->pipeline_mul_mv_f32_f32];nrows = 4;} break;@@ -1248,102 +1640,77 @@ void ggml_metal_graph_compute({nth0 = 32;nth1 = 1;- if (ne11 * ne12 < 4) {- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_1row];- } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_l4];- nrows = ne11;+ if (src1t == GGML_TYPE_F32) {+ if (ne11 * ne12 < 4) {+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_1row];+ } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_l4];+ nrows = ne11;+ } else {+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32];+ nrows = 4;+ }} else {- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32];+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f16];nrows = 4;}} break;case GGML_TYPE_Q4_0:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 8;nth1 = 8;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_0_f32];} break;case GGML_TYPE_Q4_1:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 8;nth1 = 8;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_1_f32];} break;case GGML_TYPE_Q5_0:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 8;nth1 = 8;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_0_f32];} break;case GGML_TYPE_Q5_1:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 8;nth1 = 8;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_1_f32];} break;case GGML_TYPE_Q8_0:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 8;nth1 = 8;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q8_0_f32];} break;case GGML_TYPE_Q2_K:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 2;nth1 = 32;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q2_K_f32];} break;case GGML_TYPE_Q3_K:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 2;nth1 = 32;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q3_K_f32];} break;case GGML_TYPE_Q4_K:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 4; //1;nth1 = 8; //32;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_K_f32];} break;case GGML_TYPE_Q5_K:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 2;nth1 = 32;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_K_f32];} break;case GGML_TYPE_Q6_K:{- GGML_ASSERT(ne02 == 1);- GGML_ASSERT(ne12 == 1);-nth0 = 2;nth1 = 32;[encoder setComputePipelineState:ctx->pipeline_mul_mv_q6_K_f32];@@ -1372,31 +1739,281 @@ void ggml_metal_graph_compute([encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14];[encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15];[encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];- [encoder setBytes:&gqa length:sizeof(gqa) atIndex:17];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:17];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:18];if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 ||src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 ||src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}else if (src0t == GGML_TYPE_Q4_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}else if (src0t == GGML_TYPE_Q3_K) {#ifdef GGML_QKK_64- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];#else- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];#endif}else if (src0t == GGML_TYPE_Q5_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}else if (src0t == GGML_TYPE_Q6_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ } else {+ const int64_t ny = (ne11 + nrows - 1)/nrows;+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ }+ } break;+ case GGML_OP_MUL_MAT_ID:+ {+ //GGML_ASSERT(ne00 == ne10);+ //GGML_ASSERT(ne03 == ne13);++ GGML_ASSERT(src0t == GGML_TYPE_I32);++ const int n_as = ((int32_t *) dst->op_params)[1];++ // TODO: make this more general+ GGML_ASSERT(n_as <= 8);++ struct ggml_tensor * src2 = gf->nodes[i]->src[2];++ const int64_t ne20 = src2 ? src2->ne[0] : 0;+ const int64_t ne21 = src2 ? src2->ne[1] : 0;+ const int64_t ne22 = src2 ? src2->ne[2] : 0;+ const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);++ const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);+ const uint64_t nb21 = src2 ? src2->nb[1] : 0;+ const uint64_t nb22 = src2 ? src2->nb[2] : 0;+ const uint64_t nb23 = src2 ? src2->nb[3] : 0; GGML_UNUSED(nb23);++ const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);++ GGML_ASSERT(!ggml_is_transposed(src2));+ GGML_ASSERT(!ggml_is_transposed(src1));++ GGML_ASSERT(ne20 % 32 == 0);+ // !!!!!!!!! TODO: this assert is probably required but not sure!+ //GGML_ASSERT(ne20 >= 64);+ GGML_ASSERT(src1t == GGML_TYPE_F32);++ const uint r2 = ne12/ne22;+ const uint r3 = ne13/ne23;++ // find the break-even point where the matrix-matrix kernel becomes more efficient compared+ // to the matrix-vector kernel+ int ne11_mm_min = 1;++ const int idx = ((int32_t *) dst->op_params)[0];++ // batch size+ GGML_ASSERT(ne01 == ne11);++ const int64_t _ne1 = 1; // kernel_mul_mm_impl needs a reference in constant memory++ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel+ // !!!+ // TODO: for now, always use mat-vec kernels until we figure out how to improve the+ // indirect matrix multiplication+ // !!!+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && _ne1 > ne11_mm_min) {+ switch (src2->type) {+ case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f32_f32]; break;+ case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f16_f32]; break;+ case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_0_f32]; break;+ case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_1_f32]; break;+ case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_0_f32]; break;+ case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_1_f32]; break;+ case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q8_0_f32]; break;+ case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q2_K_f32]; break;+ case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q3_K_f32]; break;+ case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_K_f32]; break;+ case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_K_f32]; break;+ case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q6_K_f32]; break;+ default: GGML_ASSERT(false && "MUL_MAT_ID not implemented");+ }+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];+ [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:5];+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];+ [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:7];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:8];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:9];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:10];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13];+ [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:14];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:16];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:17];+ [encoder setBytes:&idx length:sizeof(idx) atIndex:18];+ // TODO: how to make this an array? read Metal docs+ for (int j = 0; j < n_as; ++j) {+ struct ggml_tensor * src_cur = dst->src[2 + j];++ size_t offs_src_cur = 0;+ id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);++ [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j];+ }++ [encoder setThreadgroupMemoryLength:8192 atIndex:0];++ // TODO: processing one row at a time (ne11 -> 1) is not efficient+ [encoder dispatchThreadgroups:MTLSizeMake( (_ne1 + 31)/32, (ne21 + 63)/64, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];+ } else {+ int nth0 = 32;+ int nth1 = 1;+ int nrows = 1;+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);++ // use custom matrix x vector kernel+ switch (src2t) {+ case GGML_TYPE_F32:+ {+ GGML_ASSERT(src1t == GGML_TYPE_F32);+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f32_f32];+ } break;+ case GGML_TYPE_F16:+ {+ GGML_ASSERT(src1t == GGML_TYPE_F32);+ nth0 = 32;+ nth1 = 1;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f16_f32];+ } break;+ case GGML_TYPE_Q4_0:+ {+ nth0 = 8;+ nth1 = 8;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_0_f32];+ } break;+ case GGML_TYPE_Q4_1:+ {+ nth0 = 8;+ nth1 = 8;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_1_f32];+ } break;+ case GGML_TYPE_Q5_0:+ {+ nth0 = 8;+ nth1 = 8;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_0_f32];+ } break;+ case GGML_TYPE_Q5_1:+ {+ nth0 = 8;+ nth1 = 8;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_1_f32];+ } break;+ case GGML_TYPE_Q8_0:+ {+ nth0 = 8;+ nth1 = 8;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q8_0_f32];+ } break;+ case GGML_TYPE_Q2_K:+ {+ nth0 = 2;+ nth1 = 32;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q2_K_f32];+ } break;+ case GGML_TYPE_Q3_K:+ {+ nth0 = 2;+ nth1 = 32;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q3_K_f32];+ } break;+ case GGML_TYPE_Q4_K:+ {+ nth0 = 4; //1;+ nth1 = 8; //32;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_K_f32];+ } break;+ case GGML_TYPE_Q5_K:+ {+ nth0 = 2;+ nth1 = 32;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_K_f32];+ } break;+ case GGML_TYPE_Q6_K:+ {+ nth0 = 2;+ nth1 = 32;+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q6_K_f32];+ } break;+ default:+ {+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);+ GGML_ASSERT(false && "not implemented");+ }+ };++ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];+ [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];+ [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:6];+ [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:7];+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:8];+ [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:9];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];+ [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:11];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17];+ [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:18];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:20];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:21];+ [encoder setBytes:&idx length:sizeof(idx) atIndex:22];+ // TODO: how to make this an array? read Metal docs+ for (int j = 0; j < n_as; ++j) {+ struct ggml_tensor * src_cur = dst->src[2 + j];++ size_t offs_src_cur = 0;+ id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);++ [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j];+ }++ if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 ||+ src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 ||+ src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_Q4_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_Q3_K) {+#ifdef GGML_QKK_64+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+#else+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+#endif+ }+ else if (src2t == GGML_TYPE_Q5_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_Q6_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];} else {- int64_t ny = (ne11 + nrows - 1)/nrows;- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ const int64_t ny = (_ne1 + nrows - 1)/nrows;+ [encoder dispatchThreadgroups:MTLSizeMake(ne21, ny, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}}} break;@@ -1418,16 +2035,19 @@ void ggml_metal_graph_compute(default: GGML_ASSERT(false && "not implemented");}- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];[encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];[encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:5];-- const int64_t n = ggml_nelements(src1);-- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5];+ [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6];+ [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7];+ [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8];+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10];++ [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];} break;case GGML_OP_RMS_NORM:{@@ -1436,15 +2056,19 @@ void ggml_metal_graph_compute(float eps;memcpy(&eps, dst->op_params, sizeof(float));- const int nth = MIN(512, ne00);+ int nth = 32; // SIMD width++ while (nth < ne00/4 && nth < 1024) {+ nth *= 2;+ }[encoder setComputePipelineState:ctx->pipeline_rms_norm];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];- [encoder setBytes:&eps length:sizeof( float) atIndex:4];- [encoder setThreadgroupMemoryLength:nth/32*sizeof(float) atIndex:0];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];+ [encoder setBytes:&eps length:sizeof( float) atIndex:4];+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];const int64_t nrows = ggml_nrows(src0);@@ -1463,7 +2087,7 @@ void ggml_metal_graph_compute([encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];[encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];[encoder setBytes:&eps length:sizeof( float) atIndex:4];- [encoder setThreadgroupMemoryLength:MAX(16, nth*sizeof(float)) atIndex:0];+ [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0];const int64_t nrows = ggml_nrows(src0);@@ -1567,18 +2191,100 @@ void ggml_metal_graph_compute([encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];} break;+ case GGML_OP_IM2COL:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F16);++ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;++ const int32_t N = src1->ne[is_2D ? 3 : 2];+ const int32_t IC = src1->ne[is_2D ? 2 : 1];+ const int32_t IH = is_2D ? src1->ne[1] : 1;+ const int32_t IW = src1->ne[0];++ const int32_t KH = is_2D ? src0->ne[1] : 1;+ const int32_t KW = src0->ne[0];++ const int32_t OH = is_2D ? dst->ne[2] : 1;+ const int32_t OW = dst->ne[1];++ const int32_t CHW = IC * KH * KW;++ const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4;+ const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4;++ switch (src0->type) {+ case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break;+ case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_im2col_f16]; break;+ default: GGML_ASSERT(false);+ };++ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2];+ [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3];+ [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4];+ [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5];+ [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6];+ [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7];+ [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8];+ [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9];+ [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10];+ [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11];+ [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12];++ [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)];+ } break;+ case GGML_OP_ARGSORT:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_I32);++ const int nrows = ggml_nrows(src0);++ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];++ switch (order) {+ case GGML_SORT_ASC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_asc]; break;+ case GGML_SORT_DESC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_desc]; break;+ default: GGML_ASSERT(false);+ };++ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];++ [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00, 1, 1)];+ } break;case GGML_OP_DUP:case GGML_OP_CPY:case GGML_OP_CONT:{- const int nth = MIN(1024, ne00);+ GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);++ int nth = MIN(1024, ne00/ggml_blck_size(src0->type));switch (src0t) {case GGML_TYPE_F32:{+ GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);+switch (dstt) {- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f16]; break;- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32]; break;+ case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f16]; break;+ case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32]; break;+ case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q8_0]; break;+ case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_0]; break;+ case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_1]; break;+ //case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_0]; break;+ //case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_1]; break;default: GGML_ASSERT(false && "not implemented");};} break;@@ -1586,7 +2292,7 @@ void ggml_metal_graph_compute({switch (dstt) {case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f16]; break;- case GGML_TYPE_F32: GGML_ASSERT(false && "cpy_f16_f32 not implemented"); break;+ case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f32]; break;default: GGML_ASSERT(false && "not implemented");};} break;@@ -1616,7 +2322,7 @@ void ggml_metal_graph_compute(} break;default:{- GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %d not implemented\n", __func__, i, dst->op);+ GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));GGML_ASSERT(false);}}@@ -1648,3 +2354,239 @@ void ggml_metal_graph_compute(}}++////////////////////////////////////////////////////////////////////////////////++// backend interface++static id<MTLDevice> g_backend_device = nil;+static int g_backend_device_ref_count = 0;++static id<MTLDevice> ggml_backend_metal_get_device(void) {+ if (g_backend_device == nil) {+ g_backend_device = MTLCreateSystemDefaultDevice();+ }++ g_backend_device_ref_count++;++ return g_backend_device;+}++static void ggml_backend_metal_free_device(void) {+ assert(g_backend_device_ref_count > 0);++ g_backend_device_ref_count--;++ if (g_backend_device_ref_count == 0) {+ [g_backend_device release];+ g_backend_device = nil;+ }+}++static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;++ return ctx->data;+}++static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;++ [ctx->metal release];+ ggml_backend_metal_free_device();++ free(ctx->data);+ free(ctx);++ UNUSED(buffer);+}++static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");++ memcpy((char *)tensor->data + offset, data, size);++ UNUSED(buffer);+}++static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");++ memcpy(data, (const char *)tensor->data + offset, size);++ UNUSED(buffer);+}++static void ggml_backend_metal_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {+ ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));++ UNUSED(buffer);+}++static void ggml_backend_metal_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {+ ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));++ UNUSED(buffer);+}++static struct ggml_backend_buffer_i metal_backend_buffer_i = {+ /* .free_buffer = */ ggml_backend_metal_buffer_free_buffer,+ /* .get_base = */ ggml_backend_metal_buffer_get_base,+ /* .init_tensor = */ NULL,+ /* .set_tensor = */ ggml_backend_metal_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_metal_buffer_get_tensor,+ /* .cpy_tensor_from = */ ggml_backend_metal_buffer_cpy_tensor_from,+ /* .cpy_tensor_to = */ ggml_backend_metal_buffer_cpy_tensor_to,+};++static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));++ const size_t size_page = sysconf(_SC_PAGESIZE);++ size_t size_aligned = size;+ if ((size_aligned % size_page) != 0) {+ size_aligned += (size_page - (size_aligned % size_page));+ }++ ctx->data = ggml_metal_host_malloc(size);+ ctx->metal = [ggml_backend_metal_get_device() newBufferWithBytesNoCopy:ctx->data+ length:size_aligned+ options:MTLResourceStorageModeShared+ deallocator:nil];++ return ggml_backend_buffer_init(buft, metal_backend_buffer_i, ctx, size);+}++static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+ return 32;+ UNUSED(buft);+}++static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ return ggml_backend_is_metal(backend) || ggml_backend_is_cpu(backend);++ GGML_UNUSED(buft);+}++ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {+ static struct ggml_backend_buffer_type ggml_backend_buffer_type_metal = {+ /* .iface = */ {+ /* .alloc_buffer = */ ggml_backend_metal_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes+ /* .supports_backend = */ ggml_backend_metal_buffer_type_supports_backend,+ },+ /* .context = */ NULL,+ };++ return &ggml_backend_buffer_type_metal;+}++static const char * ggml_backend_metal_name(ggml_backend_t backend) {+ return "Metal";++ UNUSED(backend);+}++static void ggml_backend_metal_free(ggml_backend_t backend) {+ struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;+ ggml_metal_free(ctx);+ free(backend);+}++static void ggml_backend_metal_synchronize(ggml_backend_t backend) {+ UNUSED(backend);+}++static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) {+ return ggml_backend_metal_buffer_type();++ UNUSED(backend);+}++static void ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+ struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;++ ggml_metal_graph_compute(metal_ctx, cgraph);+}++static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {+ return ggml_metal_supports_op(op);++ UNUSED(backend);+}++static struct ggml_backend_i metal_backend_i = {+ /* .get_name = */ ggml_backend_metal_name,+ /* .free = */ ggml_backend_metal_free,+ /* .get_default_buffer_type = */ ggml_backend_metal_get_default_buffer_type,+ /* .set_tensor_async = */ NULL,+ /* .get_tensor_async = */ NULL,+ /* .cpy_tensor_from_async = */ NULL,+ /* .cpy_tensor_to_async = */ NULL,+ /* .synchronize = */ ggml_backend_metal_synchronize,+ /* .graph_plan_create = */ NULL, // the metal implementation does not require creating graph plans atm+ /* .graph_plan_free = */ NULL,+ /* .graph_plan_compute = */ NULL,+ /* .graph_compute = */ ggml_backend_metal_graph_compute,+ /* .supports_op = */ ggml_backend_metal_supports_op,+};++// TODO: make a common log callback for all backends in ggml-backend+static void ggml_backend_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {+ fprintf(stderr, "%s", msg);++ UNUSED(level);+ UNUSED(user_data);+}++ggml_backend_t ggml_backend_metal_init(void) {+ ggml_metal_log_set_callback(ggml_backend_log_callback, NULL);++ struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);++ if (ctx == NULL) {+ return NULL;+ }++ ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend));++ *metal_backend = (struct ggml_backend) {+ /* .interface = */ metal_backend_i,+ /* .context = */ ctx,+ };++ return metal_backend;+}++bool ggml_backend_is_metal(ggml_backend_t backend) {+ return backend->iface.get_name == ggml_backend_metal_name;+}++void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {+ GGML_ASSERT(ggml_backend_is_metal(backend));++ struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;++ ggml_metal_set_n_cb(ctx, n_cb);+}++bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {+ GGML_ASSERT(ggml_backend_is_metal(backend));++ struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;++ return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];+}++ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning++ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {+ return ggml_backend_metal_init();++ GGML_UNUSED(params);+ GGML_UNUSED(user_data);+}diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metalindex 7c35f23a..773fac12 100644--- a/llama.cpp/ggml-metal.metal+++ b/llama.cpp/ggml-metal.metal@@ -3,6 +3,8 @@using namespace metal;#define MAX(x, y) ((x) > (y) ? (x) : (y))+#define MIN(x, y) ((x) < (y) ? (x) : (y))+#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }#define QK4_0 32#define QR4_0 2@@ -39,8 +41,15 @@ typedef struct {int8_t qs[QK8_0]; // quants} block_q8_0;-// general-purpose kernel for addition of two tensors-// pros: works for non-contiguous tensors, supports broadcast across dims 1, 2 and 3+#define N_SIMDWIDTH 32 // assuming SIMD group size is 32++enum ggml_sort_order {+ GGML_SORT_ASC,+ GGML_SORT_DESC,+};++// general-purpose kernel for addition, multiplication and division of two tensors+// pros: works for non-contiguous tensors, supports broadcast across all dims// cons: not very efficientkernel void kernel_add(device const char * src0,@@ -81,16 +90,111 @@ kernel void kernel_add(const int64_t i12 = i02 % ne12;const int64_t i11 = i01 % ne11;- device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + tpitg.x*nb00;- device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11 + tpitg.x*nb10;- device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + tpitg.x*nb0;+ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ const int i10 = i0 % ne10;+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) + *((device float *)(src1_ptr + i10*nb10));+ }+}++kernel void kernel_mul(+ device const char * src0,+ device const char * src1,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant int64_t & nb00,+ constant int64_t & nb01,+ constant int64_t & nb02,+ constant int64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant int64_t & nb10,+ constant int64_t & nb11,+ constant int64_t & nb12,+ constant int64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant int64_t & nb0,+ constant int64_t & nb1,+ constant int64_t & nb2,+ constant int64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig.z;+ const int64_t i02 = tgpig.y;+ const int64_t i01 = tgpig.x;++ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;++ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- ((device float *)dst_ptr)[0] = ((device float *)src0_ptr)[0] + ((device float *)src1_ptr)[0];+ const int i10 = i0 % ne10;+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) * *((device float *)(src1_ptr + i10*nb10));+ }+}++kernel void kernel_div(+ device const char * src0,+ device const char * src1,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant int64_t & nb00,+ constant int64_t & nb01,+ constant int64_t & nb02,+ constant int64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant int64_t & nb10,+ constant int64_t & nb11,+ constant int64_t & nb12,+ constant int64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant int64_t & nb0,+ constant int64_t & nb1,+ constant int64_t & nb2,+ constant int64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig.z;+ const int64_t i02 = tgpig.y;+ const int64_t i01 = tgpig.x;++ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;++ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;- src0_ptr += ntg.x*nb00;- src1_ptr += ntg.x*nb10;- dst_ptr += ntg.x*nb0;+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ const int i10 = i0 % ne10;+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) / *((device float *)(src1_ptr + i10*nb10));}}@@ -105,23 +209,22 @@ kernel void kernel_add_row(dst[tpig] = src0[tpig] + src1[tpig % nb];}-kernel void kernel_mul(+kernel void kernel_mul_row(device const float4 * src0,device const float4 * src1,device float4 * dst,+ constant int64_t & nb [[buffer(27)]],uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] * src1[tpig];+ dst[tpig] = src0[tpig] * src1[tpig % nb];}-// assumption: src1 is a row-// broadcast src1 into src0-kernel void kernel_mul_row(+kernel void kernel_div_row(device const float4 * src0,device const float4 * src1,device float4 * dst,- constant int64_t & nb,+ constant int64_t & nb [[buffer(27)]],uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] * src1[tpig % nb];+ dst[tpig] = src0[tpig] / src1[tpig % nb];}kernel void kernel_scale(@@ -162,6 +265,54 @@ kernel void kernel_sqr(dst[tpig] = src0[tpig] * src0[tpig];}+kernel void kernel_sum_rows(+ device const float * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant int64_t & nb00,+ constant int64_t & nb01,+ constant int64_t & nb02,+ constant int64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant int64_t & nb10,+ constant int64_t & nb11,+ constant int64_t & nb12,+ constant int64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant int64_t & nb0,+ constant int64_t & nb1,+ constant int64_t & nb2,+ constant int64_t & nb3,+ uint3 tpig[[thread_position_in_grid]]) {+ int64_t i3 = tpig.z;+ int64_t i2 = tpig.y;+ int64_t i1 = tpig.x;++ if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {+ return;+ }++ device const float * src_row = (device const float *) ((device const char *) src0 + i1*nb01 + i2*nb02 + i3*nb03);+ device float * dst_row = (device float *) ((device char *) dst + i1*nb1 + i2*nb2 + i3*nb3);++ float row_sum = 0;++ for (int64_t i0 = 0; i0 < ne00; i0++) {+ row_sum += src_row[i0];+ }++ dst_row[0] = row_sum;+}+constant float GELU_COEF_A = 0.044715f;constant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;@@ -180,10 +331,12 @@ kernel void kernel_gelu(kernel void kernel_soft_max(device const float * src0,+ device const float * src1,device float * dst,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,+ constant float & scale,threadgroup float * buf [[threadgroup(0)]],uint tgpig[[threadgroup_position_in_grid]],uint tpitg[[thread_position_in_threadgroup]],@@ -194,73 +347,82 @@ kernel void kernel_soft_max(const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);- device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;- device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;+ device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;+ device const float * pmask = src1 != src0 ? src1 + i01*ne00 : nullptr;+ device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;// parallel max- float lmax = tpitg < ne00 ? psrc0[tpitg] : -INFINITY;+ float lmax = -INFINITY;- for (int i00 = tpitg + ntg; i00 < ne00; i00 += ntg) {- lmax = MAX(lmax, psrc0[i00]);+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {+ lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f));}- float max = simd_max(lmax);- if (tiisg == 0) {- buf[sgitg] = max;- }+ // find the max value in the block+ float max_val = simd_max(lmax);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = -INFINITY;+ }- threadgroup_barrier(mem_flags::mem_threadgroup);+ threadgroup_barrier(mem_flags::mem_threadgroup);- // broadcast, simd group number is ntg / 32- for (uint i = ntg / 32 / 2; i > 0; i /= 2) {- if (tpitg < i) {- buf[tpitg] = MAX(buf[tpitg], buf[tpitg + i]);- }- }+ if (tiisg == 0) {+ buf[sgitg] = max_val;+ }- threadgroup_barrier(mem_flags::mem_threadgroup);+ threadgroup_barrier(mem_flags::mem_threadgroup);- max = buf[0];+ max_val = buf[tiisg];+ max_val = simd_max(max_val);+ }// parallel sumfloat lsum = 0.0f;for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- const float exp_psrc0 = exp(psrc0[i00] - max);+ const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f)) - max_val);lsum += exp_psrc0;- // Remember the result of exp here. exp is expensive, so we really do not- // wish to compute it twice.pdst[i00] = exp_psrc0;}+ // This barrier fixes a failing test+ // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335+ threadgroup_barrier(mem_flags::mem_none);+float sum = simd_sum(lsum);- if (tiisg == 0) {- buf[sgitg] = sum;- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }- // broadcast, simd group number is ntg / 32- for (uint i = ntg / 32 / 2; i > 0; i /= 2) {- if (tpitg < i) {- buf[tpitg] += buf[tpitg + i];- }- }+ threadgroup_barrier(mem_flags::mem_threadgroup);- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tiisg == 0) {+ buf[sgitg] = sum;+ }- sum = buf[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);++ sum = buf[tiisg];+ sum = simd_sum(sum);+ }++ const float inv_sum = 1.0f/sum;for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- pdst[i00] /= sum;+ pdst[i00] *= inv_sum;}}kernel void kernel_soft_max_4(device const float * src0,+ device const float * src1,device float * dst,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,+ constant float & scale,threadgroup float * buf [[threadgroup(0)]],uint tgpig[[threadgroup_position_in_grid]],uint tpitg[[thread_position_in_threadgroup]],@@ -271,64 +433,74 @@ kernel void kernel_soft_max_4(const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);- device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);- device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);+ device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);+ device const float4 * pmask = src1 != src0 ? (device const float4 *)(src1 + i01*ne00) : nullptr;+ device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);// parallel max- float4 lmax4 = tpitg < ne00/4 ? psrc4[tpitg] : -INFINITY;+ float4 lmax4 = -INFINITY;- for (int i00 = tpitg + ntg; i00 < ne00/4; i00 += ntg) {- lmax4 = fmax(lmax4, psrc4[i00]);+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f));}const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));- float max = simd_max(lmax);- if (tiisg == 0) {- buf[sgitg] = max;- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ float max_val = simd_max(lmax);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = -INFINITY;+ }- // broadcast, simd group number is ntg / 32- for (uint i = ntg / 32 / 2; i > 0; i /= 2) {- if (tpitg < i) {- buf[tpitg] = MAX(buf[tpitg], buf[tpitg + i]);- }- }+ threadgroup_barrier(mem_flags::mem_threadgroup);- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tiisg == 0) {+ buf[sgitg] = max_val;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);- max = buf[0];+ max_val = buf[tiisg];+ max_val = simd_max(max_val);+ }// parallel sumfloat4 lsum4 = 0.0f;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- const float4 exp_psrc4 = exp(psrc4[i00] - max);+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f)) - max_val);lsum4 += exp_psrc4;pdst4[i00] = exp_psrc4;}const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];++ // This barrier fixes a failing test+ // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335+ threadgroup_barrier(mem_flags::mem_none);+float sum = simd_sum(lsum);- if (tiisg == 0) {- buf[sgitg] = sum;- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }- // broadcast, simd group number is ntg / 32- for (uint i = ntg / 32 / 2; i > 0; i /= 2) {- if (tpitg < i) {- buf[tpitg] += buf[tpitg + i];- }- }+ threadgroup_barrier(mem_flags::mem_threadgroup);- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tiisg == 0) {+ buf[sgitg] = sum;+ }- sum = buf[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);++ sum = buf[tiisg];+ sum = simd_sum(sum);+ }++ const float inv_sum = 1.0f/sum;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- pdst4[i00] /= sum;+ pdst4[i00] *= inv_sum;}}@@ -435,14 +607,13 @@ kernel void kernel_rms_norm(constant int64_t & ne00,constant uint64_t & nb01,constant float & eps,- threadgroup float * sum [[threadgroup(0)]],+ threadgroup float * buf [[threadgroup(0)]],uint tgpig[[threadgroup_position_in_grid]],uint tpitg[[thread_position_in_threadgroup]],uint sgitg[[simdgroup_index_in_threadgroup]],uint tiisg[[thread_index_in_simdgroup]],uint ntg[[threads_per_threadgroup]]) {- device const float4 * x = (device const float4 *) ((device const char *) src0 + tgpig*nb01);- device const float * x_scalar = (device const float *) x;+ device const float4 * x = (device const float4 *) ((device const char *) src0 + tgpig*nb01);float4 sumf = 0;float all_sum = 0;@@ -453,40 +624,30 @@ kernel void kernel_rms_norm(}all_sum = sumf[0] + sumf[1] + sumf[2] + sumf[3];all_sum = simd_sum(all_sum);- if (tiisg == 0) {- sum[sgitg] = all_sum;- }+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }- threadgroup_barrier(mem_flags::mem_threadgroup);+ threadgroup_barrier(mem_flags::mem_threadgroup);- // broadcast, simd group number is ntg / 32- for (uint i = ntg / 32 / 2; i > 0; i /= 2) {- if (tpitg < i) {- sum[tpitg] += sum[tpitg + i];- }- }- if (tpitg == 0) {- for (int i = 4 * (ne00 / 4); i < ne00; i++) {- sum[0] += x_scalar[i];+ if (tiisg == 0) {+ buf[sgitg] = all_sum;}- sum[0] /= ne00;- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ threadgroup_barrier(mem_flags::mem_threadgroup);++ all_sum = buf[tiisg];+ all_sum = simd_sum(all_sum);+ }- const float mean = sum[0];+ const float mean = all_sum/ne00;const float scale = 1.0f/sqrt(mean + eps);device float4 * y = (device float4 *) (dst + tgpig*ne00);- device float * y_scalar = (device float *) y;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {y[i00] = x[i00] * scale;}- if (tpitg == 0) {- for (int i00 = 4 * (ne00 / 4); i00 < ne00; i00++) {- y_scalar[i00] = x_scalar[i00] * scale;- }- }}// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])@@ -576,15 +737,25 @@ inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thre// putting them in the kernel cause a significant performance penalty#define N_DST 4 // each SIMD group works on 4 rows#define N_SIMDGROUP 2 // number of SIMD groups in a thread group-#define N_SIMDWIDTH 32 // assuming SIMD group size is 32//Note: This is a template, but strictly speaking it only applies to// quantizations where the block size is 32. It also does not// giard against the number of rows not being divisible by// N_DST, so this is another explicit assumption of the implementation.template<typename block_q_type, int nr, int nsg, int nw>-void mul_vec_q_n_f32(device const void * src0, device const float * src1, device float * dst,- int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne10, int64_t ne12, int64_t ne0, int64_t ne1, uint gqa,- uint3 tgpig, uint tiisg, uint sgitg) {+void mul_vec_q_n_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ int64_t ne00,+ int64_t ne01,+ int64_t ne02,+ int64_t ne10,+ int64_t ne12,+ int64_t ne0,+ int64_t ne1,+ uint r2,+ uint r3,+ uint3 tgpig, uint tiisg, uint sgitg) {const int nb = ne00/QK4_0;const int r0 = tgpig.x;@@ -593,7 +764,10 @@ void mul_vec_q_n_f32(device const void * src0, device const float * src1, deviceconst int first_row = (r0 * nsg + sgitg) * nr;- const uint offset0 = first_row * nb + im/gqa*(nb*ne0);+ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);device const block_q_type * x = (device const block_q_type *) src0 + offset0;device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;@@ -643,13 +817,14 @@ kernel void kernel_mul_mv_q4_0_f32(constant int64_t & ne02[[buffer(5)]],constant int64_t & ne10[[buffer(9)]],constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,gqa,tgpig,tiisg,sgitg);+ mul_vec_q_n_f32_impl<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);}kernel void kernel_mul_mv_q4_1_f32(@@ -661,13 +836,14 @@ kernel void kernel_mul_mv_q4_1_f32(constant int64_t & ne02[[buffer(5)]],constant int64_t & ne10[[buffer(9)]],constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,gqa,tgpig,tiisg,sgitg);+ mul_vec_q_n_f32_impl<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);}kernel void kernel_mul_mv_q5_0_f32(@@ -679,13 +855,14 @@ kernel void kernel_mul_mv_q5_0_f32(constant int64_t & ne02[[buffer(5)]],constant int64_t & ne10[[buffer(9)]],constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,gqa,tgpig,tiisg,sgitg);+ mul_vec_q_n_f32_impl<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);}kernel void kernel_mul_mv_q5_1_f32(@@ -697,33 +874,35 @@ kernel void kernel_mul_mv_q5_1_f32(constant int64_t & ne02[[buffer(5)]],constant int64_t & ne10[[buffer(9)]],constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,gqa,tgpig,tiisg,sgitg);+ mul_vec_q_n_f32_impl<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);}#define NB_Q8_0 8-kernel void kernel_mul_mv_q8_0_f32(+void kernel_mul_mv_q8_0_f32_impl(device const void * src0,device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {const int nr = N_DST;const int nsg = N_SIMDGROUP;const int nw = N_SIMDWIDTH;@@ -732,8 +911,14 @@ kernel void kernel_mul_mv_q8_0_f32(const int r0 = tgpig.x;const int r1 = tgpig.y;const int im = tgpig.z;+const int first_row = (r0 * nsg + sgitg) * nr;- const uint offset0 = first_row * nb + im/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q8_0 * x = (device const block_q8_0 *) src0 + offset0;device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;@@ -771,9 +956,29 @@ kernel void kernel_mul_mv_q8_0_f32(}}+[[host_name("kernel_mul_mv_q8_0_f32")]]+kernel void kernel_mul_mv_q8_0_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ kernel_mul_mv_q8_0_f32_impl(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);+}+#define N_F32_F32 4-kernel void kernel_mul_mv_f32_f32(+void kernel_mul_mv_f32_f32_impl(device const char * src0,device const char * src1,device float * dst,@@ -791,16 +996,23 @@ kernel void kernel_mul_mv_f32_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]]) {const int64_t r0 = tgpig.x;const int64_t rb = tgpig.y*N_F32_F32;const int64_t im = tgpig.z;- device const float * x = (device const float *) (src0 + r0*nb01 + im/(ne12/ne02)*nb02);+ const uint i12 = im%ne12;+ const uint i13 = im/ne12;- if (ne00 < 128) {+ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const float * x = (device const float *) (src0 + offset0);++ if (ne00 < 128) {for (int row = 0; row < N_F32_F32; ++row) {int r1 = rb + row;if (r1 >= ne11) {@@ -844,7 +1056,8 @@ kernel void kernel_mul_mv_f32_f32(}}-kernel void kernel_mul_mv_f16_f32_1row(+[[host_name("kernel_mul_mv_f32_f32")]]+kernel void kernel_mul_mv_f32_f32(device const char * src0,device const char * src1,device float * dst,@@ -862,6 +1075,113 @@ kernel void kernel_mul_mv_f16_f32_1row(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {+ kernel_mul_mv_f32_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);+}++#define N_F16_F16 4++kernel void kernel_mul_mv_f16_f16(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {++ const int64_t r0 = tgpig.x;+ const int64_t rb = tgpig.y*N_F16_F16;+ const int64_t im = tgpig.z;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half * x = (device const half *) (src0 + offset0);++ if (ne00 < 128) {+ for (int row = 0; row < N_F16_F16; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const half * y = (device const half *) (src1 + r1*nb11 + im*nb12);++ float sumf = 0;+ for (int i = tiisg; i < ne00; i += 32) {+ sumf += (half) x[i] * (half) y[i];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ } else {+ device const half4 * x4 = (device const half4 *)x;+ for (int row = 0; row < N_F16_F16; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const half * y = (device const half *) (src1 + r1*nb11 + im*nb12);+ device const half4 * y4 = (device const half4 *) y;++ float sumf = 0;+ for (int i = tiisg; i < ne00/4; i += 32) {+ for (int k = 0; k < 4; ++k) sumf += (half) x4[i][k] * y4[i][k];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (half) x[i] * y[i];+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ }+}++void kernel_mul_mv_f16_f32_1row_impl(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {@@ -869,7 +1189,12 @@ kernel void kernel_mul_mv_f16_f32_1row(const int64_t r1 = tgpig.y;const int64_t im = tgpig.z;- device const half * x = (device const half *) (src0 + r0*nb01 + im/(ne12/ne02)*nb02);+ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half * x = (device const half *) (src0 + offset0);device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);float sumf = 0;@@ -893,12 +1218,37 @@ kernel void kernel_mul_mv_f16_f32_1row(dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;}}+}+[[host_name("kernel_mul_mv_f16_f32_1row")]]+kernel void kernel_mul_mv_f16_f32_1row(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {+ kernel_mul_mv_f16_f32_1row_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);}#define N_F16_F32 4-kernel void kernel_mul_mv_f16_f32(+void kernel_mul_mv_f16_f32_impl(device const char * src0,device const char * src1,device float * dst,@@ -916,6 +1266,8 @@ kernel void kernel_mul_mv_f16_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {@@ -923,7 +1275,12 @@ kernel void kernel_mul_mv_f16_f32(const int64_t rb = tgpig.y*N_F16_F32;const int64_t im = tgpig.z;- device const half * x = (device const half *) (src0 + r0*nb01 + im/(ne12/ne02)*nb02);+ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half * x = (device const half *) (src0 + offset0);if (ne00 < 128) {for (int row = 0; row < N_F16_F32; ++row) {@@ -969,6 +1326,32 @@ kernel void kernel_mul_mv_f16_f32(}}+[[host_name("kernel_mul_mv_f16_f32")]]+kernel void kernel_mul_mv_f16_f32(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {+ kernel_mul_mv_f16_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);+}+// Assumes row size (ne00) is a multiple of 4kernel void kernel_mul_mv_f16_f32_l4(device const char * src0,@@ -988,6 +1371,8 @@ kernel void kernel_mul_mv_f16_f32_l4(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {@@ -995,7 +1380,12 @@ kernel void kernel_mul_mv_f16_f32_l4(const int64_t r0 = tgpig.x;const int64_t im = tgpig.z;- device const half4 * x4 = (device const half4 *) (src0 + r0*nb01 + im/(ne12/ne02)*nb02);+ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half4 * x4 = (device const half4 *) (src0 + offset0);for (int r1 = 0; r1 < nrows; ++r1) {device const float4 * y4 = (device const float4 *) (src1 + r1*nb11 + im*nb12);@@ -1047,17 +1437,21 @@ kernel void kernel_alibi_f32(const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);+ const int64_t k = i3*ne3 + i2;- device float * dst_data = (device float *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);float m_k;- if (i2 < n_heads_log2_floor) {- m_k = pow(m0, i2 + 1);+ if (k < n_heads_log2_floor) {+ m_k = pow(m0, k + 1);} else {- m_k = pow(m1, 2 * (i2 - n_heads_log2_floor) + 1);+ m_k = pow(m1, 2 * (k - n_heads_log2_floor) + 1);}++ device char * dst_row = (device char *) dst + i3*nb3 + i2*nb2 + i1*nb1;+ device const char * src_row = (device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01;for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {- device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);- dst_data[i00] = src[0] + m_k * (i00 - ne00 + 1);+ const float src_v = *(device float *)(src_row + i00*nb00);+ device float * dst_v = (device float *)(dst_row + i00*nb0);+ *dst_v = i00 * m_k + src_v;}}@@ -1229,9 +1623,94 @@ kernel void kernel_rope(template [[host_name("kernel_rope_f32")]] kernel rope_t kernel_rope<float>;template [[host_name("kernel_rope_f16")]] kernel rope_t kernel_rope<half>;-kernel void kernel_cpy_f16_f16(- device const half * src0,+kernel void kernel_im2col_f16(+ device const float * x,device half * dst,+ constant int32_t & ofs0,+ constant int32_t & ofs1,+ constant int32_t & IW,+ constant int32_t & IH,+ constant int32_t & CHW,+ constant int32_t & s0,+ constant int32_t & s1,+ constant int32_t & p0,+ constant int32_t & p1,+ constant int32_t & d0,+ constant int32_t & d1,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tgpg[[threadgroups_per_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int32_t iiw = tgpig[2] * s0 + tpitg[2] * d0 - p0;+ const int32_t iih = tgpig[1] * s1 + tpitg[1] * d1 - p1;++ const int32_t offset_dst =+ (tpitg[0] * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * CHW ++ (tgpig[0] * (ntg[1] * ntg[2]) + tpitg[1] * ntg[2] + tpitg[2]);++ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ dst[offset_dst] = 0.0f;+ } else {+ const int32_t offset_src = tpitg[0] * ofs0 + tgpig[0] * ofs1;+ dst[offset_dst] = x[offset_src + iih * IW + iiw];+ }+}++// bitonic sort implementation following the CUDA kernels as reference+typedef void (argsort_t)(+ device const float * x,+ device int32_t * dst,+ constant int64_t & ncols,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]]);++template<ggml_sort_order order>+kernel void kernel_argsort_f32_i32(+ device const float * x,+ device int32_t * dst,+ constant int64_t & ncols,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]]) {+ // bitonic sort+ int col = tpitg[0];+ int row = tgpig[1];++ if (col >= ncols) return;++ device const float * x_row = x + row * ncols;+ device int32_t * dst_row = dst + row * ncols;++ // initialize indices+ if (col < ncols) {+ dst_row[col] = col;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (int k = 2; k <= ncols; k *= 2) {+ for (int j = k / 2; j > 0; j /= 2) {+ int ixj = col ^ j;+ if (ixj > col) {+ if ((col & k) == 0) {+ if (order == GGML_SORT_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) {+ SWAP(dst_row[col], dst_row[ixj]);+ }+ } else {+ if (order == GGML_SORT_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) {+ SWAP(dst_row[col], dst_row[ixj]);+ }+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }+}++template [[host_name("kernel_argsort_f32_i32_asc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_ASC>;+template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_DESC>;++kernel void kernel_cpy_f16_f16(+ device const half * src0,+ device half * dst,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -1270,6 +1749,47 @@ kernel void kernel_cpy_f16_f16(}}+kernel void kernel_cpy_f16_f32(+ device const half * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);++ device float * dst_data = (device float *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {+ device const half * src = (device half *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);+ dst_data[i00] = src[0];+ }+}+kernel void kernel_cpy_f32_f16(device const float * src0,device half * dst,@@ -1354,6 +1874,197 @@ kernel void kernel_cpy_f32_f32(}}+kernel void kernel_cpy_f32_q8_0(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK8_0;++ device block_q8_0 * dst_data = (device block_q8_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK8_0; i00 < ne00; i00 += ntg.x*QK8_0) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK8_0; j++) {+ const float v = src[j];+ amax = MAX(amax, fabs(v));+ }++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK8_0].d = d;++ for (int j = 0; j < QK8_0; ++j) {+ const float x0 = src[j]*id;++ dst_data[i00/QK8_0].qs[j] = round(x0);+ }+ }+}++kernel void kernel_cpy_f32_q4_0(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_0;++ device block_q4_0 * dst_data = (device block_q4_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK4_0; i00 < ne00; i00 += ntg.x*QK4_0) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float amax = 0.0f; // absolute max+ float max = 0.0f;++ for (int j = 0; j < QK4_0; j++) {+ const float v = src[j];+ if (amax < fabs(v)) {+ amax = fabs(v);+ max = v;+ }+ }++ const float d = max / -8;+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK4_0].d = d;++ for (int j = 0; j < QK4_0/2; ++j) {+ const float x0 = src[0 + j]*id;+ const float x1 = src[QK4_0/2 + j]*id;++ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));++ dst_data[i00/QK4_0].qs[j] = xi0;+ dst_data[i00/QK4_0].qs[j] |= xi1 << 4;+ }+ }+}++kernel void kernel_cpy_f32_q4_1(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_1;++ device block_q4_1 * dst_data = (device block_q4_1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK4_1; i00 < ne00; i00 += ntg.x*QK4_1) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float min = FLT_MAX;+ float max = -FLT_MAX;++ for (int j = 0; j < QK4_1; j++) {+ const float v = src[j];+ if (min > v) min = v;+ if (max < v) max = v;+ }++ const float d = (max - min) / ((1 << 4) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK4_1].d = d;+ dst_data[i00/QK4_1].m = min;++ for (int j = 0; j < QK4_1/2; ++j) {+ const float x0 = (src[0 + j] - min)*id;+ const float x1 = (src[QK4_1/2 + j] - min)*id;++ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));++ dst_data[i00/QK4_1].qs[j] = xi0;+ dst_data[i00/QK4_1].qs[j] |= xi1 << 4;+ }+ }+}+kernel void kernel_concat(device const char * src0,device const char * src1,@@ -1502,32 +2213,39 @@ static inline uchar4 get_scale_min_k4(int j, device const uint8_t * q) {//====================================== dot products =========================-kernel void kernel_mul_mv_q2_K_f32(+void kernel_mul_mv_q2_K_f32_impl(device const void * src0,device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {const int nb = ne00/QK_K;const int r0 = tgpig.x;const int r1 = tgpig.y;- const int r2 = tgpig.z;+ const int im = tgpig.z;const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;const int ib_row = first_row * nb;- const uint offset0 = r2/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q2_K * x = (device const block_q2_K *) src0 + ib_row + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + r2*ne00*ne1;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;+float yl[32];float sumf[N_DST]={0.f}, all_sum;@@ -1536,11 +2254,11 @@ kernel void kernel_mul_mv_q2_K_f32(#if QK_K == 256const int ix = tiisg/8; // 0...3const int it = tiisg%8; // 0...7- const int im = it/4; // 0 or 1+ const int iq = it/4; // 0 or 1const int ir = it%4; // 0...3const int is = (8*ir)/16;// 0 or 1- device const float * y4 = y + ix * QK_K + 128 * im + 8 * ir;+ device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;for (int ib = ix; ib < nb; ib += 4) {@@ -1552,8 +2270,8 @@ kernel void kernel_mul_mv_q2_K_f32(yl[i+24] = y4[i+96]; sumy[3] += yl[i+24];}- device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*im + is;- device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * im + 4 * ir;+ device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is;+ device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;device const half * dh = &x[ib].d;for (int row = 0; row < N_DST; row++) {@@ -1640,13 +2358,13 @@ kernel void kernel_mul_mv_q2_K_f32(for (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);if (tiisg == 0) {- dst[r1*ne0 + r2*ne0*ne1 + first_row + row] = all_sum;+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;}}}-#if QK_K == 256-kernel void kernel_mul_mv_q3_K_f32(+[[host_name("kernel_mul_mv_q2_K_f32")]]+kernel void kernel_mul_mv_q2_K_f32(device const void * src0,device const float * src1,device float * dst,@@ -1655,23 +2373,50 @@ kernel void kernel_mul_mv_q3_K_f32(constant int64_t & ne02[[buffer(5)]],constant int64_t & ne10[[buffer(9)]],constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q2_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}++#if QK_K == 256+void kernel_mul_mv_q3_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {const int nb = ne00/QK_K;const int64_t r0 = tgpig.x;const int64_t r1 = tgpig.y;- const int64_t r2 = tgpig.z;+ const int64_t im = tgpig.z;const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;- const uint offset0 = r2/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q3_K * x = (device const block_q3_K *) src0 + first_row*nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + r2*ne00*ne1;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;float yl[32];@@ -1793,40 +2538,47 @@ kernel void kernel_mul_mv_q3_K_f32(}if (tiisg == 0) {for (int row = 0; row < 2; ++row) {- dst[r1*ne0 + r2*ne0*ne1 + first_row + row] = sumf1[row];+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = sumf1[row];}}}#else-kernel void kernel_mul_mv_q3_K_f32(+void kernel_mul_mv_q3_K_f32_impl(device const void * src0,device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {const int nb = ne00/QK_K;const int64_t r0 = tgpig.x;const int64_t r1 = tgpig.y;- const int64_t r2 = tgpig.z;+ const int64_t im = tgpig.z;const int row = 2 * r0 + sgitg;- const uint offset0 = r2/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q3_K * x = (device const block_q3_K *) src0 + row*nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + r2*ne00*ne1;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;+const int ix = tiisg/4;const int il = 4 * (tiisg%4);// 0, 4, 8, 12- const int im = il/8; // 0, 0, 1, 1+ const int iq = il/8; // 0, 0, 1, 1const int in = il%8; // 0, 4, 0, 4float2 sum = {0.f, 0.f};@@ -1846,7 +2598,7 @@ kernel void kernel_mul_mv_q3_K_f32(const float d4 = d_all * ((int32_t)(s[0] & 0xF000) - 32768) * 1.f/262144.f;for (int l = 0; l < 4; l += 2) {- const uint16_t hm = h[l/2] >> im;+ const uint16_t hm = h[l/2] >> iq;sum[0] += y[l+ 0] * d1 * ((int32_t)(q[l/2] & 0x0003) - ((hm & 0x0001) ? 0 : 4))+ y[l+16] * d2 * ((int32_t)(q[l/2] & 0x000c) - ((hm & 0x0004) ? 0 : 16))+ y[l+32] * d3 * ((int32_t)(q[l/2] & 0x0030) - ((hm & 0x0010) ? 0 : 64))@@ -1862,28 +2614,50 @@ kernel void kernel_mul_mv_q3_K_f32(const float tot = simd_sum(sumf);if (tiisg == 0) {- dst[r1*ne0 + r2*ne0*ne1 + row] = tot;+ dst[r1*ne0 + im*ne0*ne1 + row] = tot;}}#endif+[[host_name("kernel_mul_mv_q3_K_f32")]]+kernel void kernel_mul_mv_q3_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01[[buffer(4)]],+ constant int64_t & ne02[[buffer(5)]],+ constant int64_t & ne10[[buffer(9)]],+ constant int64_t & ne12[[buffer(11)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q3_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}+#if QK_K == 256-kernel void kernel_mul_mv_q4_K_f32(+void kernel_mul_mv_q4_K_f32_impl(device const void * src0,device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01 [[buffer(4)]],- constant int64_t & ne02 [[buffer(5)]],- constant int64_t & ne10 [[buffer(9)]],- constant int64_t & ne12 [[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & gqa [[buffer(17)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {const uint16_t kmask1 = 0x3f3f;const uint16_t kmask2 = 0x0f0f;@@ -1891,26 +2665,32 @@ kernel void kernel_mul_mv_q4_K_f32(const int ix = tiisg/8; // 0...3const int it = tiisg%8; // 0...7- const int im = it/4; // 0 or 1+ const int iq = it/4; // 0 or 1const int ir = it%4; // 0...3const int nb = ne00/QK_K;const int r0 = tgpig.x;const int r1 = tgpig.y;- const int r2 = tgpig.z;+ const int im = tgpig.z;//const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;const int first_row = r0 * N_DST;const int ib_row = first_row * nb;- const uint offset0 = r2/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + r2*ne00*ne1;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;+float yl[16];float yh[16];float sumf[N_DST]={0.f}, all_sum;const int step = sizeof(block_q4_K) * nb / 2;- device const float * y4 = y + ix * QK_K + 64 * im + 8 * ir;+ device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;uint16_t sc16[4];thread const uint8_t * sc8 = (thread const uint8_t *)sc16;@@ -1925,8 +2705,8 @@ kernel void kernel_mul_mv_q4_K_f32(yh[i+8] = y4[i+160]; sumy[3] += yh[i+8];}- device const uint16_t * sc = (device const uint16_t *)x[ib].scales + im;- device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * im + 4 * ir;+ device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq;+ device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;device const half * dh = &x[ib].d;for (int row = 0; row < N_DST; row++) {@@ -1970,23 +2750,24 @@ kernel void kernel_mul_mv_q4_K_f32(for (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);if (tiisg == 0) {- dst[r1*ne0 + r2*ne0*ne1 + first_row + row] = all_sum;+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;}}}#else-kernel void kernel_mul_mv_q4_K_f32(+void kernel_mul_mv_q4_K_f32_impl(device const void * src0,device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -1997,12 +2778,18 @@ kernel void kernel_mul_mv_q4_K_f32(const int nb = ne00/QK_K;const int r0 = tgpig.x;const int r1 = tgpig.y;- const int r2 = tgpig.z;+ const int im = tgpig.z;const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;const int ib_row = first_row * nb;- const uint offset0 = r2/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + r2*ne00*ne1;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;+float yl[8];float yh[8];float sumf[N_DST]={0.f}, all_sum;@@ -2058,13 +2845,14 @@ kernel void kernel_mul_mv_q4_K_f32(for (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);if (tiisg == 0) {- dst[r1*ne0+ r2*ne0*ne1 + first_row + row] = all_sum;+ dst[r1*ne0+ im*ne0*ne1 + first_row + row] = all_sum;}}}#endif-kernel void kernel_mul_mv_q5_K_f32(+[[host_name("kernel_mul_mv_q4_K_f32")]]+kernel void kernel_mul_mv_q4_K_f32(device const void * src0,device const float * src1,device float * dst,@@ -2073,23 +2861,49 @@ kernel void kernel_mul_mv_q5_K_f32(constant int64_t & ne02[[buffer(5)]],constant int64_t & ne10[[buffer(9)]],constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {+ kernel_mul_mv_q4_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}++void kernel_mul_mv_q5_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+const int nb = ne00/QK_K;const int64_t r0 = tgpig.x;const int64_t r1 = tgpig.y;- const int r2 = tgpig.z;+ const int im = tgpig.z;const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;- const uint offset0 = r2/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q5_K * x = (device const block_q5_K *) src0 + first_row*nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + r2*ne00*ne1;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;float sumf[2]={0.f};@@ -2105,15 +2919,15 @@ kernel void kernel_mul_mv_q5_K_f32(const int tid = tiisg/4;const int ix = tiisg%4;- const int im = tid/4;+ const int iq = tid/4;const int ir = tid%4;const int n = 8;const int l0 = n*ir;- const int q_offset = 32*im + l0;- const int y_offset = 64*im + l0;+ const int q_offset = 32*iq + l0;+ const int y_offset = 64*iq + l0;- const uint8_t hm1 = 1u << (2*im);+ const uint8_t hm1 = 1u << (2*iq);const uint8_t hm2 = hm1 << 1;const uint8_t hm3 = hm1 << 4;const uint8_t hm4 = hm2 << 4;@@ -2128,7 +2942,7 @@ kernel void kernel_mul_mv_q5_K_f32(device const uint8_t * q1 = x[i].qs + q_offset;device const uint8_t * qh = x[i].qh + l0;device const half * dh = &x[i].d;- device const uint16_t * a = (device const uint16_t *)x[i].scales + im;+ device const uint16_t * a = (device const uint16_t *)x[i].scales + iq;device const float * y2 = y1 + 128;float4 sumy = {0.f, 0.f, 0.f, 0.f};@@ -2184,7 +2998,7 @@ kernel void kernel_mul_mv_q5_K_f32(const int il = 4 * (tiisg/8); // 0, 4, 8, 12const int ix = tiisg%8;- const int im = il/8; // 0, 0, 1, 1+ const int iq = il/8; // 0, 0, 1, 1const int in = il%8; // 0, 4, 0, 4device const float * y = yy + ix*QK_K + il;@@ -2209,7 +3023,7 @@ kernel void kernel_mul_mv_q5_K_f32(float2 acc = {0.f, 0.f};for (int l = 0; l < 4; ++l) {- const uint8_t hl = h[l] >> im;+ const uint8_t hl = h[l] >> iq;acc[0] += yl[l+0] * s[0] * ((int16_t)(q[l+ 0] & 0x0F) - (hl & 0x01 ? 0 : 16))+ yl[l+4] * s[1] * ((int16_t)(q[l+16] & 0x0F) - (hl & 0x04 ? 0 : 16));acc[1] += yh[l+0] * s[2] * ((int16_t)(q[l+ 0] & 0xF0) - (hl & 0x10 ? 0 : 256))@@ -2231,27 +3045,48 @@ kernel void kernel_mul_mv_q5_K_f32(for (int row = 0; row < 2; ++row) {const float tot = simd_sum(sumf[row]);if (tiisg == 0) {- dst[r1*ne0 + r2*ne0*ne1 + first_row + row] = tot;+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;}}+}++[[host_name("kernel_mul_mv_q5_K_f32")]]+kernel void kernel_mul_mv_q5_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01[[buffer(4)]],+ constant int64_t & ne02[[buffer(5)]],+ constant int64_t & ne10[[buffer(9)]],+ constant int64_t & ne12[[buffer(11)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ kernel_mul_mv_q5_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);}-kernel void kernel_mul_mv_q6_K_f32(+void kernel_mul_mv_q6_K_f32_impl(device const void * src0,device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0[[buffer(15)]],- constant int64_t & ne1[[buffer(16)]],- constant uint & gqa[[buffer(17)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {const uint8_t kmask1 = 0x03;const uint8_t kmask2 = 0x0C;@@ -2262,12 +3097,17 @@ kernel void kernel_mul_mv_q6_K_f32(const int64_t r0 = tgpig.x;const int64_t r1 = tgpig.y;- const int r2 = tgpig.z;+ const int im = tgpig.z;const int row = 2 * r0 + sgitg;- const uint offset0 = r2/gqa*(nb*ne0);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+device const block_q6_K * x = (device const block_q6_K *) src0 + row * nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + r2*ne00*ne1;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;float sumf = 0;@@ -2333,10 +3173,31 @@ kernel void kernel_mul_mv_q6_K_f32(const float tot = simd_sum(sumf);if (tiisg == 0) {- dst[r1*ne0 + r2*ne0*ne1 + row] = tot;+ dst[r1*ne0 + im*ne0*ne1 + row] = tot;}}+[[host_name("kernel_mul_mv_q6_K_f32")]]+kernel void kernel_mul_mv_q6_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01[[buffer(4)]],+ constant int64_t & ne02[[buffer(5)]],+ constant int64_t & ne10[[buffer(9)]],+ constant int64_t & ne12[[buffer(11)]],+ constant int64_t & ne0 [[buffer(15)]],+ constant int64_t & ne1 [[buffer(16)]],+ constant uint & r2 [[buffer(17)]],+ constant uint & r3 [[buffer(18)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q6_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}+//============================= templates and their specializations =============================// NOTE: this is not dequantizing - we are simply fitting the template@@ -2611,22 +3472,90 @@ void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 & regtemplate<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>kernel void kernel_get_rows(device const void * src0,- device const int * src1,+ device const char * src1,device float * dst,constant int64_t & ne00,constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant uint64_t & nb10,+ constant uint64_t & nb11,constant uint64_t & nb1,- uint tgpig[[threadgroup_position_in_grid]],+ constant uint64_t & nb2,+ uint3 tgpig[[threadgroup_position_in_grid]],uint tiitg[[thread_index_in_threadgroup]],- uint tptg[[threads_per_threadgroup]]) {- const int i = tgpig;- const int r = ((device int32_t *) src1)[i];+ uint3 tptg [[threads_per_threadgroup]]) {+ //const int64_t i = tgpig;+ //const int64_t r = ((device int32_t *) src1)[i];++ const int64_t i10 = tgpig.x;+ const int64_t i11 = tgpig.y;- for (int ind = tiitg; ind < ne00/16; ind += tptg) {+ const int64_t r = ((device int32_t *) ((device char *) src1 + i11*nb11 + i10*nb10))[0];++ const int64_t i02 = i11;++ for (int64_t ind = tiitg; ind < ne00/16; ind += tptg.x) {float4x4 temp;dequantize_func(- ((device const block_q *) ((device char *) src0 + r*nb01)) + ind/nl, ind%nl, temp);- *(((device float4x4 *) ((device char *) dst + i*nb1)) + ind) = temp;+ ((device const block_q *) ((device char *) src0 + r*nb01 + i02*nb02)) + ind/nl, ind%nl, temp);+ *(((device float4x4 *) ((device char *) dst + i11*nb2 + i10*nb1)) + ind) = temp;+ }+}++kernel void kernel_get_rows_f32(+ device const void * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint3 tptg [[threads_per_threadgroup]]) {+ const int64_t i10 = tgpig.x;+ const int64_t i11 = tgpig.y;++ const int64_t r = ((device int32_t *) ((device char *) src1 + i11*nb11 + i10*nb10))[0];++ const int64_t i02 = i11;++ for (int ind = tiitg; ind < ne00; ind += tptg.x) {+ ((device float *) ((device char *) dst + i11*nb2 + i10*nb1))[ind] =+ ((device float *) ((device char *) src0 + r*nb01 + i02*nb02))[ind];+ }+}++kernel void kernel_get_rows_f16(+ device const void * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint3 tptg [[threads_per_threadgroup]]) {+ const int64_t i10 = tgpig.x;+ const int64_t i11 = tgpig.y;++ const int64_t r = ((device int32_t *) ((device char *) src1 + i11*nb11 + i10*nb10))[0];++ const int64_t i02 = i11;++ for (int ind = tiitg; ind < ne00; ind += tptg.x) {+ ((device float *) ((device char *) dst + i11*nb2 + i10*nb1))[ind] =+ ((device half *) ((device char *) src0 + r*nb01 + i02*nb02))[ind];}}@@ -2643,24 +3572,25 @@ kernel void kernel_get_rows(// each block_q contains 16*nl weightstemplate<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread half4x4 &)>-kernel void kernel_mul_mm(device const uchar * src0,- device const uchar * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne02,- constant int64_t & nb01,- constant int64_t & nb02,- constant int64_t & ne12,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & gqa,- threadgroup uchar * shared_memory [[threadgroup(0)]],- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiitg[[thread_index_in_threadgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+void kernel_mul_mm_impl(device const uchar * src0,+ device const uchar * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne02,+ constant int64_t & nb01,+ constant int64_t & nb02,+ constant int64_t & ne12,+ constant int64_t & nb10,+ constant int64_t & nb11,+ constant int64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup uchar * shared_memory [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {threadgroup half * sa = (threadgroup half *)(shared_memory);threadgroup float * sb = (threadgroup float *)(shared_memory + 4096);@@ -2686,7 +3616,10 @@ kernel void kernel_mul_mm(device const uchar * src0,short il = (tiitg % THREAD_PER_ROW);- uint offset0 = im/gqa*nb02;+ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ uint offset0 = (i12/r2)*nb02 + (i13/r3)*(nb02*ne02);ushort offset1 = il/nl;device const block_q * x = (device const block_q *)(src0 + (r0 * BLOCK_SIZE_M + thread_row) * nb01 + offset0) + offset1;@@ -2770,17 +3703,137 @@ kernel void kernel_mul_mm(device const uchar * src0,}}+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread half4x4 &)>+kernel void kernel_mul_mm(device const uchar * src0,+ device const uchar * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne02,+ constant int64_t & nb01,+ constant int64_t & nb02,+ constant int64_t & ne12,+ constant int64_t & nb10,+ constant int64_t & nb11,+ constant int64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup uchar * shared_memory [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ kernel_mul_mm_impl<block_q, nl, dequantize_func>(+ src0,+ src1,+ dst,+ ne00,+ ne02,+ nb01,+ nb02,+ ne12,+ nb10,+ nb11,+ nb12,+ ne0,+ ne1,+ r2,+ r3,+ shared_memory,+ tgpig,+ tiitg,+ sgitg);+}++template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread half4x4 &)>+kernel void kernel_mul_mm_id(+ device const uchar * ids,+ device const uchar * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne02,+ constant int64_t & nb01,+ constant int64_t & nb02,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant int64_t & nb10,+ constant int64_t & nb11,+ constant int64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const uchar * src00,+ device const uchar * src01,+ device const uchar * src02,+ device const uchar * src03,+ device const uchar * src04,+ device const uchar * src05,+ device const uchar * src06,+ device const uchar * src07,+ threadgroup uchar * shared_memory [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const uchar * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mm_impl<block_q, nl, dequantize_func>(+ src0[id],+ src1 + bid*nb11,+ (device float *) (dst + bid*nb1),+ ne00,+ ne02,+ nb01,+ nb02,+ ne12,+ nb10,+ nb11,+ nb12,+ ne0,+ ne1,+ r2,+ r3,+ shared_memory,+ tgpig,+ tiitg,+ sgitg);+}+#if QK_K == 256#define QK_NL 16#else#define QK_NL 4#endif-typedef void (get_rows_t)(device const void *, device const int *, device float *, constant int64_t &, \- constant uint64_t &, constant uint64_t &, uint, uint, uint);+//+// get rows+//-template [[host_name("kernel_get_rows_f32")]] kernel get_rows_t kernel_get_rows<float4x4, 1, dequantize_f32>;-template [[host_name("kernel_get_rows_f16")]] kernel get_rows_t kernel_get_rows<half4x4, 1, dequantize_f16>;+typedef void (get_rows_t)(+ device const void * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ uint3, uint, uint3);++//template [[host_name("kernel_get_rows_f32")]] kernel get_rows_t kernel_get_rows<float4x4, 1, dequantize_f32>;+//template [[host_name("kernel_get_rows_f16")]] kernel get_rows_t kernel_get_rows<half4x4, 1, dequantize_f16>;template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_t kernel_get_rows<block_q4_0, 2, dequantize_q4_0>;template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_t kernel_get_rows<block_q4_1, 2, dequantize_q4_1>;template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_t kernel_get_rows<block_q5_0, 2, dequantize_q5_0>;@@ -2792,6 +3845,10 @@ template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_t kernel_get_rowstemplate [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_t kernel_get_rows<block_q5_K, QK_NL, dequantize_q5_K>;template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_t kernel_get_rows<block_q6_K, QK_NL, dequantize_q6_K>;+//+// matrix-matrix multiplication+//+typedef void (mat_mm_t)(device const uchar * src0,device const uchar * src1,@@ -2806,8 +3863,10 @@ typedef void (mat_mm_t)(constant int64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant uint & gqa,- threadgroup uchar *, uint3, uint, uint);+ constant uint & r2,+ constant uint & r3,+ threadgroup uchar *,+ uint3, uint, uint);template [[host_name("kernel_mul_mm_f32_f32")]] kernel mat_mm_t kernel_mul_mm<float4x4, 1, dequantize_f32>;template [[host_name("kernel_mul_mm_f16_f32")]] kernel mat_mm_t kernel_mul_mm<half4x4, 1, dequantize_f16>;@@ -2821,3 +3880,823 @@ template [[host_name("kernel_mul_mm_q3_K_f32")]] kernel mat_mm_t kernel_mul_mm<btemplate [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mat_mm_t kernel_mul_mm<block_q4_K, QK_NL, dequantize_q4_K>;template [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mat_mm_t kernel_mul_mm<block_q5_K, QK_NL, dequantize_q5_K>;template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm<block_q6_K, QK_NL, dequantize_q6_K>;++//+// indirect matrix-matrix multiplication+//++typedef void (mat_mm_id_t)(+ device const uchar * ids,+ device const uchar * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne02,+ constant int64_t & nb01,+ constant int64_t & nb02,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant int64_t & nb10,+ constant int64_t & nb11,+ constant int64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const uchar * src00,+ device const uchar * src01,+ device const uchar * src02,+ device const uchar * src03,+ device const uchar * src04,+ device const uchar * src05,+ device const uchar * src06,+ device const uchar * src07,+ threadgroup uchar *,+ uint3, uint, uint);++template [[host_name("kernel_mul_mm_id_f32_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<float4x4, 1, dequantize_f32>;+template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<half4x4, 1, dequantize_f16>;+template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q4_0, 2, dequantize_q4_0>;+template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q4_1, 2, dequantize_q4_1>;+template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q5_0, 2, dequantize_q5_0>;+template [[host_name("kernel_mul_mm_id_q5_1_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q5_1, 2, dequantize_q5_1>;+template [[host_name("kernel_mul_mm_id_q8_0_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q8_0, 2, dequantize_q8_0>;+template [[host_name("kernel_mul_mm_id_q2_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q2_K, QK_NL, dequantize_q2_K>;+template [[host_name("kernel_mul_mm_id_q3_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q3_K, QK_NL, dequantize_q3_K>;+template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q4_K, QK_NL, dequantize_q4_K>;+template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q5_K, QK_NL, dequantize_q5_K>;+template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q6_K, QK_NL, dequantize_q6_K>;++//+// matrix-vector multiplication+//++[[host_name("kernel_mul_mv_id_f32_f32")]]+kernel void kernel_mul_mv_id_f32_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_f32_f32_impl(+ src0[id],+ src1 + bid*nb11,+ (device float *) (dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ nb00,+ nb01,+ nb02,+ ne10,+ ne11,+ ne12,+ nb10,+ nb11,+ nb12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg);+}++[[host_name("kernel_mul_mv_id_f16_f32")]]+kernel void kernel_mul_mv_id_f16_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_f16_f32_impl(+ src0[id],+ src1 + bid*nb11,+ (device float *) (dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ nb00,+ nb01,+ nb02,+ ne10,+ ne11,+ ne12,+ nb10,+ nb11,+ nb12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg);+}++[[host_name("kernel_mul_mv_id_q8_0_f32")]]+kernel void kernel_mul_mv_id_q8_0_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_q8_0_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q4_0_f32")]]+kernel void kernel_mul_mv_id_q4_0_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ mul_vec_q_n_f32_impl<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q4_1_f32")]]+kernel void kernel_mul_mv_id_q4_1_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ mul_vec_q_n_f32_impl<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q5_0_f32")]]+kernel void kernel_mul_mv_id_q5_0_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ mul_vec_q_n_f32_impl<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q5_1_f32")]]+kernel void kernel_mul_mv_id_q5_1_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ mul_vec_q_n_f32_impl<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q2_K_f32")]]+kernel void kernel_mul_mv_id_q2_K_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_q2_K_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q3_K_f32")]]+kernel void kernel_mul_mv_id_q3_K_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_q3_K_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q4_K_f32")]]+kernel void kernel_mul_mv_id_q4_K_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_q4_K_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q5_K_f32")]]+kernel void kernel_mul_mv_id_q5_K_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_q5_K_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_q6_K_f32")]]+kernel void kernel_mul_mv_id_q6_K_f32(+ device const char * ids,+ device const char * src1,+ device uchar * dst,+ constant int64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_q6_K_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ (device float *) ( dst + bid*nb1),+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex 96abc22a..b6d39088 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -17,32 +17,12 @@//#include <arm_neon.h>-#if !defined(__aarch64__)-inline static int32_t vaddvq_s16(int16x8_t v) {- return- (int32_t)vgetq_lane_s16(v, 0) + (int32_t)vgetq_lane_s16(v, 1) +- (int32_t)vgetq_lane_s16(v, 2) + (int32_t)vgetq_lane_s16(v, 3) +- (int32_t)vgetq_lane_s16(v, 4) + (int32_t)vgetq_lane_s16(v, 5) +- (int32_t)vgetq_lane_s16(v, 6) + (int32_t)vgetq_lane_s16(v, 7);-}--inline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) {- int16x4_t a0 = vpadd_s16(vget_low_s16(a), vget_high_s16(a));- int16x4_t b0 = vpadd_s16(vget_low_s16(b), vget_high_s16(b));- return vcombine_s16(a0, b0);-}--inline static int32_t vaddvq_s32(int32x4_t v) {- return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3);-}-#endif-#else#ifdef __wasm_simd128__#include <wasm_simd128.h>#else-#ifdef __POWER9_VECTOR__+#if defined(__POWER9_VECTOR__) || defined(__powerpc64__)#include <altivec.h>#undef bool#define bool _Bool@@ -50,13 +30,15 @@ inline static int32_t vaddvq_s32(int32x4_t v) {#if defined(_MSC_VER) || defined(__MINGW32__)#include <intrin.h>#else-#if !defined(__riscv) && !defined(__s390__)+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) || defined(__SSE3__)+#if !defined(__riscv)#include <immintrin.h>#endif#endif#endif#endif#endif+#endif#ifdef __riscv_v_intrinsic#include <riscv_vector.h>@@ -64,6 +46,7 @@ inline static int32_t vaddvq_s32(int32x4_t v) {#undef MIN#undef MAX+#define MIN(a, b) ((a) < (b) ? (a) : (b))#define MAX(a, b) ((a) > (b) ? (a) : (b))@@ -162,6 +145,19 @@ static inline __m256 sum_i16_pairs_float_avx2(const __m256i x) {return _mm256_cvtepi32_ps(summed_pairs);}+__attribute__((__target__("avx2,fma")))+static inline __m256 mul_sum_us8_pairs_float_avx2(const __m256i ax, const __m256i sy) {+#if __AVXVNNI__+ const __m256i zero = _mm256_setzero_si256();+ const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);+ return _mm256_cvtepi32_ps(summed_pairs);+#else+ // Perform multiplication and create 16-bit values+ const __m256i dot = _mm256_maddubs_epi16(ax, sy);+ return sum_i16_pairs_float_avx2(dot);+#endif+}+__attribute__((__target__("avx")))static inline __m256 mul_sum_us8_pairs_float_avx(const __m256i ax, const __m256i sy) {const __m128i axl = _mm256_castsi256_si128(ax);@@ -174,16 +170,19 @@ static inline __m256 mul_sum_us8_pairs_float_avx(const __m256i ax, const __m256ireturn sum_i16_pairs_float_avx(doth, dotl);}+// multiply int8_t, add results pairwise twice and return as float vector__attribute__((__target__("avx2,fma")))-static inline __m256 mul_sum_us8_pairs_float_avx2(const __m256i ax, const __m256i sy) {-#if __AVXVNNI__+static inline __m256 mul_sum_i8_pairs_float_avx2(const __m256i x, const __m256i y) {+#if __AVXVNNIINT8__const __m256i zero = _mm256_setzero_si256();- const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);+ const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);return _mm256_cvtepi32_ps(summed_pairs);#else- // Perform multiplication and create 16-bit values- const __m256i dot = _mm256_maddubs_epi16(ax, sy);- return sum_i16_pairs_float_avx2(dot);+ // Get absolute values of x vectors+ const __m256i ax = _mm256_sign_epi8(x, x);+ // Sign the values of the y vectors+ const __m256i sy = _mm256_sign_epi8(y, x);+ return mul_sum_us8_pairs_float_avx2(ax, sy);#endif}@@ -206,22 +205,6 @@ static inline __m256 mul_sum_i8_pairs_float_avx(const __m256i x, const __m256i yreturn sum_i16_pairs_float_avx(doth, dotl);}-// multiply int8_t, add results pairwise twice and return as float vector-__attribute__((__target__("avx2,fma")))-static inline __m256 mul_sum_i8_pairs_float_avx2(const __m256i x, const __m256i y) {-#if __AVXVNNIINT8__- const __m256i zero = _mm256_setzero_si256();- const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);- return _mm256_cvtepi32_ps(summed_pairs);-#else- // Get absolute values of x vectors- const __m256i ax = _mm256_sign_epi8(x, x);- // Sign the values of the y vectors- const __m256i sy = _mm256_sign_epi8(y, x);- return mul_sum_us8_pairs_float_avx2(ax, sy);-#endif-}-#endif /* __x86_64__ */#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)@@ -238,6 +221,14 @@ static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {}#if __AVX__ || __AVX2__ || __AVX512F__+// horizontally add 8 floats+static inline float hsum_float_8(const __m256 x) {+ __m128 res = _mm256_extractf128_ps(x, 1);+ res = _mm_add_ps(res, _mm256_castps256_ps128(x));+ res = _mm_add_ps(res, _mm_movehl_ps(res, res));+ res = _mm_add_ss(res, _mm_movehdup_ps(res));+ return _mm_cvtss_f32(res);+}// horizontally add 8 int32_tstatic inline int hsum_i32_8(const __m256i a) {@@ -257,7 +248,6 @@ static inline int hsum_i32_4(const __m128i a) {}#if defined(__AVX2__) || defined(__AVX512F__)-static inline __m128i packNibbles( __m256i bytes ){// Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh@@ -279,7 +269,6 @@ static inline __m128i packNibbles( __m256i bytes )#endif}#elif defined(__AVX__)-static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 ){// Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh@@ -311,9 +300,31 @@ static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128#endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)#if defined(__ARM_NEON)-#if !defined(__aarch64__)+// 64-bit compatibility++// vaddvq_s16+// vpaddq_s16+// vaddvq_s32+// vaddvq_f32+// vmaxvq_f32+// vcvtnq_s32_f32++inline static int32_t vaddvq_s16(int16x8_t v) {+ return+ (int32_t)vgetq_lane_s16(v, 0) + (int32_t)vgetq_lane_s16(v, 1) ++ (int32_t)vgetq_lane_s16(v, 2) + (int32_t)vgetq_lane_s16(v, 3) ++ (int32_t)vgetq_lane_s16(v, 4) + (int32_t)vgetq_lane_s16(v, 5) ++ (int32_t)vgetq_lane_s16(v, 6) + (int32_t)vgetq_lane_s16(v, 7);+}++inline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) {+ int16x4_t a0 = vpadd_s16(vget_low_s16(a), vget_high_s16(a));+ int16x4_t b0 = vpadd_s16(vget_low_s16(b), vget_high_s16(b));+ return vcombine_s16(a0, b0);+}+inline static int32_t vaddvq_s32(int32x4_t v) {return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3);}@@ -339,6 +350,96 @@ inline static int32x4_t vcvtnq_s32_f32(float32x4_t v) {return res;}+// vld1q_s16_x2+// vld1q_u8_x2+// vld1q_u8_x4+// vld1q_s8_x2+// vld1q_s8_x4+// TODO: double-check these work correctly++typedef struct ggml_int16x8x2_t {+ int16x8_t val[2];+} ggml_int16x8x2_t;++inline static ggml_int16x8x2_t ggml_vld1q_s16_x2(const int16_t * ptr) {+ ggml_int16x8x2_t res;++ res.val[0] = vld1q_s16(ptr + 0);+ res.val[1] = vld1q_s16(ptr + 8);++ return res;+}++typedef struct ggml_uint8x16x2_t {+ uint8x16_t val[2];+} ggml_uint8x16x2_t;++inline static ggml_uint8x16x2_t ggml_vld1q_u8_x2(const uint8_t * ptr) {+ ggml_uint8x16x2_t res;++ res.val[0] = vld1q_u8(ptr + 0);+ res.val[1] = vld1q_u8(ptr + 16);++ return res;+}++typedef struct ggml_uint8x16x4_t {+ uint8x16_t val[4];+} ggml_uint8x16x4_t;++inline static ggml_uint8x16x4_t ggml_vld1q_u8_x4(const uint8_t * ptr) {+ ggml_uint8x16x4_t res;++ res.val[0] = vld1q_u8(ptr + 0);+ res.val[1] = vld1q_u8(ptr + 16);+ res.val[2] = vld1q_u8(ptr + 32);+ res.val[3] = vld1q_u8(ptr + 48);++ return res;+}++typedef struct ggml_int8x16x2_t {+ int8x16_t val[2];+} ggml_int8x16x2_t;++inline static ggml_int8x16x2_t ggml_vld1q_s8_x2(const int8_t * ptr) {+ ggml_int8x16x2_t res;++ res.val[0] = vld1q_s8(ptr + 0);+ res.val[1] = vld1q_s8(ptr + 16);++ return res;+}++typedef struct ggml_int8x16x4_t {+ int8x16_t val[4];+} ggml_int8x16x4_t;++inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) {+ ggml_int8x16x4_t res;++ res.val[0] = vld1q_s8(ptr + 0);+ res.val[1] = vld1q_s8(ptr + 16);+ res.val[2] = vld1q_s8(ptr + 32);+ res.val[3] = vld1q_s8(ptr + 48);++ return res;+}++#else++#define ggml_int16x8x2_t int16x8x2_t+#define ggml_uint8x16x2_t uint8x16x2_t+#define ggml_uint8x16x4_t uint8x16x4_t+#define ggml_int8x16x2_t int8x16x2_t+#define ggml_int8x16x4_t int8x16x4_t++#define ggml_vld1q_s16_x2 vld1q_s16_x2+#define ggml_vld1q_u8_x2 vld1q_u8_x2+#define ggml_vld1q_u8_x4 vld1q_u8_x4+#define ggml_vld1q_s8_x2 vld1q_s8_x2+#define ggml_vld1q_s8_x4 vld1q_s8_x4+#endif#endif@@ -1301,7 +1402,12 @@ static float make_qkx2_quants(int n, int nmax, const float * restrict x, const ffloat max = x[0];float sum_w = weights[0];float sum_x = sum_w * x[0];+#ifdef HAVE_BUGGY_APPLE_LINKER+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7+ for (volatile int i = 1; i < n; ++i) {+#elsefor (int i = 1; i < n; ++i) {+#endifif (x[i] < min) min = x[i];if (x[i] > max) max = x[i];float w = weights[i];@@ -2357,7 +2463,6 @@ static inline __m128i get_scale_shuffle(int i) {#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_default#include "ggml_vec_dot_q4_0_q8_0.inc"#undef ggml_vec_dot_q4_0_q8_0-#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q4_0_q8_0_avx#ifndef __AVX__#define __UNDEF_AVX__@@ -2369,7 +2474,6 @@ static inline __m128i get_scale_shuffle(int i) {#undef __AVX__#endif#undef ggml_vec_dot_q4_0_q8_0-#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_0_q8_0_avx2#ifndef __AVX2__#define __UNDEF_AVX2__@@ -2381,9 +2485,8 @@ static inline __m128i get_scale_shuffle(int i) {#undef __AVX2__#endif#undef ggml_vec_dot_q4_0_q8_0-void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {ggml_vec_dot_q4_0_q8_0_avx2(n, s, vx, vy);} else if (X86_HAVE(AVX)) {ggml_vec_dot_q4_0_q8_0_avx(n, s, vx, vy);@@ -2391,804 +2494,139 @@ void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void * restriggml_vec_dot_q4_0_q8_0_default(n, s, vx, vy);}}-#else#define static#include "ggml_vec_dot_q4_0_q8_0.inc"#undef static#endif-void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- const int qk = QK8_1;- const int nb = n / qk;-- assert(n % qk == 0);-- const block_q4_1 * restrict x = vx;- const block_q8_1 * restrict y = vy;-- // TODO: add WASM SIMD-#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- float summs = 0;-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q4_1 * restrict x0 = &x[i + 0];- const block_q4_1 * restrict x1 = &x[i + 1];- const block_q8_1 * restrict y0 = &y[i + 0];- const block_q8_1 * restrict y1 = &y[i + 1];-- summs += GGML_FP16_TO_FP32(x0->m) * y0->s + GGML_FP16_TO_FP32(x1->m) * y1->s;-- const uint8x16_t m4b = vdupq_n_u8(0x0F);-- const uint8x16_t v0_0 = vld1q_u8(x0->qs);- const uint8x16_t v0_1 = vld1q_u8(x1->qs);-- // 4-bit -> 8-bit- const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));-- // load y- const int8x16_t v1_0l = vld1q_s8(y0->qs);- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);- const int8x16_t v1_1l = vld1q_s8(y1->qs);- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);--#if defined(__ARM_FEATURE_DOTPROD)- // dot product into int32x4_t- const int32x4_t p_0 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);- const int32x4_t p_1 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*y0->d);- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*y1->d);-#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0l), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0l), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0h), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0h), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1l), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1l), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1h), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1h), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*y0->d);- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*y1->d);+#ifdef __x86_64__+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_default+#include "ggml_vec_dot_q4_1_q8_1.inc"+#undef ggml_vec_dot_q4_1_q8_1+#define ggml_vec_dot_q4_1_q8_1 __attribute__((__target__("avx"))) ggml_vec_dot_q4_1_q8_1_avx+#ifndef __AVX__+#define __UNDEF_AVX__+#define __AVX__#endif- }-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;-#elif defined(__AVX2__) || defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- float summs = 0;-- // Main loop- for (int i = 0; i < nb; ++i) {- const float d0 = GGML_FP16_TO_FP32(x[i].d);- const float d1 = y[i].d;-- summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;-- const __m256 d0v = _mm256_set1_ps( d0 );- const __m256 d1v = _mm256_set1_ps( d1 );-- // Compute combined scales- const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );-- // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes- const __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);- const __m256i by = _mm256_loadu_si256( (const __m256i *)y[i].qs );-- const __m256 xy = mul_sum_us8_pairs_float_avx2(bx, by);-- // Accumulate d0*d1*x*y-#if defined(__AVX2__)- acc = _mm256_fmadd_ps( d0d1, xy, acc );-#else- acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );+#include "ggml_vec_dot_q4_1_q8_1.inc"+#ifdef __UNDEF_AVX__+#undef __UNDEF_AVX__+#undef __AVX__#endif- }-- *s = hsum_float_8(acc) + summs;-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;-- size_t vl = __riscv_vsetvl_e8m1(qk/2);-- for (int i = 0; i < nb; i++) {- // load elements- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);-- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);-- // mask and store lower part of x, and then upper part- vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);- vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);-- vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);-- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);-- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);-- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);-- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);-- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;- }-- *s = sumf;-#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- int sumi = 0;-- for (int j = 0; j < qk/2; ++j) {- const int v0 = (x[i].qs[j] & 0x0F);- const int v1 = (x[i].qs[j] >> 4);-- sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);- }-- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;- }-- *s = sumf;+#undef ggml_vec_dot_q4_1_q8_1+#define ggml_vec_dot_q4_1_q8_1 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_1_q8_1_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__#endif-}--void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- const int qk = QK8_0;- const int nb = n / qk;-- assert(n % qk == 0);- assert(qk == QK5_0);-- const block_q5_0 * restrict x = vx;- const block_q8_0 * restrict y = vy;--#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- uint32_t qh0;- uint32_t qh1;-- uint64_t tmp0[4];- uint64_t tmp1[4];-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q5_0 * restrict x0 = &x[i];- const block_q5_0 * restrict x1 = &x[i + 1];- const block_q8_0 * restrict y0 = &y[i];- const block_q8_0 * restrict y1 = &y[i + 1];-- const uint8x16_t m4b = vdupq_n_u8(0x0F);-- // extract the 5th bit via lookup table ((!b) << 4)- memcpy(&qh0, x0->qh, sizeof(qh0));- memcpy(&qh1, x1->qh, sizeof(qh1));-- tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];- tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];- tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];- tmp0[3] = table_b2b_1[(qh0 >> 24) ];-- tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];- tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];- tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];- tmp1[3] = table_b2b_1[(qh1 >> 24) ];-- const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));- const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));- const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));- const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));-- const uint8x16_t v0_0 = vld1q_u8(x0->qs);- const uint8x16_t v0_1 = vld1q_u8(x1->qs);-- // 4-bit -> 8-bit- int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));-- // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)- const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);- const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);- const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);- const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);-- // load y- const int8x16_t v1_0l = vld1q_s8(y0->qs);- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);- const int8x16_t v1_1l = vld1q_s8(y1->qs);- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);--#if defined(__ARM_FEATURE_DOTPROD)- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),- vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),- vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));-#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0lf), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0lf), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hf), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hf), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1lf), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1lf), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hf), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hf), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#include "ggml_vec_dot_q4_1_q8_1.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__#endif+#undef ggml_vec_dot_q4_1_q8_1+void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {+ ggml_vec_dot_q4_1_q8_1_avx2(n, s, vx, vy);+ } else if (X86_HAVE(AVX)) {+ ggml_vec_dot_q4_1_q8_1_avx(n, s, vx, vy);+ } else {+ ggml_vec_dot_q4_1_q8_1_default(n, s, vx, vy);}-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);-#elif defined(__wasm_simd128__)- v128_t sumv = wasm_f32x4_splat(0.0f);-- uint32_t qh;- uint64_t tmp[4];-- // TODO: check if unrolling this is better- for (int i = 0; i < nb; ++i) {- const block_q5_0 * restrict x0 = &x[i];- const block_q8_0 * restrict y0 = &y[i];-- const v128_t m4b = wasm_i8x16_splat(0x0F);-- // extract the 5th bit- memcpy(&qh, x0->qh, sizeof(qh));-- tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];- tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];- tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];- tmp[3] = table_b2b_1[(qh >> 24) ];-- const v128_t qhl = wasm_v128_load(tmp + 0);- const v128_t qhh = wasm_v128_load(tmp + 2);-- const v128_t v0 = wasm_v128_load(x0->qs);-- // 4-bit -> 8-bit- const v128_t v0l = wasm_v128_and (v0, m4b);- const v128_t v0h = wasm_u8x16_shr(v0, 4);-- // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)- const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);- const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);-- // load y- const v128_t v1l = wasm_v128_load(y0->qs);- const v128_t v1h = wasm_v128_load(y0->qs + 16);-- // int8x16 -> int16x8- const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);- const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);- const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);- const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);-- const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);- const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);- const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);- const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);-- // dot product- sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(- wasm_i32x4_add(- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),- wasm_i32x4_dot_i16x8(v0lfh, v1lh)),- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),- wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),- wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));- }-- *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +- wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);-#elif defined(__AVX2__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; i++) {- /* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));-- __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);- __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);- bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));- bx = _mm256_or_si256(bx, bxhi);-- __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);-- /* Multiply q with scale and accumulate */- acc = _mm256_fmadd_ps(d, q, acc);- }-- *s = hsum_float_8(acc);-#elif defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();- __m128i mask = _mm_set1_epi8((char)0xF0);-- // Main loop- for (int i = 0; i < nb; i++) {- /* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));-- __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);- const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);- __m128i bxhil = _mm256_castsi256_si128(bxhi);- __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);- bxhil = _mm_andnot_si128(bxhil, mask);- bxhih = _mm_andnot_si128(bxhih, mask);- __m128i bxl = _mm256_castsi256_si128(bx);- __m128i bxh = _mm256_extractf128_si256(bx, 1);- bxl = _mm_or_si128(bxl, bxhil);- bxh = _mm_or_si128(bxh, bxhih);- bx = MM256_SET_M128I(bxh, bxl);-- const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_i8_pairs_float_avx(bx, by);-- /* Multiply q with scale and accumulate */- acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);- }-- *s = hsum_float_8(acc);-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;-- uint32_t qh;-- size_t vl = __riscv_vsetvl_e8m1(qk/2);-- // These tempory registers are for masking and shift operations- vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);- vuint32m2_t vt_2 = __riscv_vsll_vv_u32m2(__riscv_vmv_v_x_u32m2(1, vl), vt_1, vl);-- vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);- vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);-- for (int i = 0; i < nb; i++) {- memcpy(&qh, x[i].qh, sizeof(uint32_t));-- // ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;- vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);- vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(xha_0, vt_1, vl);- vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);-- // ((qh & (1u << (j + 16))) >> (j + 12));- vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(vt_3, qh, vl);- vuint32m2_t xhl_1 = __riscv_vsrl_vv_u32m2(xha_1, vt_4, vl);-- // narrowing- vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xhl_0, vl);- vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);-- vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xhl_1, vl);- vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);-- // load- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);-- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);-- vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);- vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);-- vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);- vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);-- vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);-- vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 16, vl);- vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 16, vl);-- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);-- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);-- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);-- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);-- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;- }-- *s = sumf;+}#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- uint32_t qh;- memcpy(&qh, x[i].qh, sizeof(qh));-- int sumi = 0;-- for (int j = 0; j < qk/2; ++j) {- const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;- const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));-- const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;- const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;-- sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);- }-- sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;- }+#define static+#include "ggml_vec_dot_q4_1_q8_1.inc"+#undef static+#endif- *s = sumf;+#ifdef __x86_64__+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_default+#include "ggml_vec_dot_q5_0_q8_0.inc"+#undef ggml_vec_dot_q5_0_q8_0+#define ggml_vec_dot_q5_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q5_0_q8_0_avx+#ifndef __AVX__+#define __UNDEF_AVX__+#define __AVX__+#endif+#include "ggml_vec_dot_q5_0_q8_0.inc"+#ifdef __UNDEF_AVX__+#undef __UNDEF_AVX__+#undef __AVX__#endif+#undef ggml_vec_dot_q5_0_q8_0+#define ggml_vec_dot_q5_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q5_0_q8_0_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__+#endif+#include "ggml_vec_dot_q5_0_q8_0.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__+#endif+#undef ggml_vec_dot_q5_0_q8_0+void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {+ ggml_vec_dot_q5_0_q8_0_avx2(n, s, vx, vy);+ } else if (X86_HAVE(AVX)) {+ ggml_vec_dot_q5_0_q8_0_avx(n, s, vx, vy);+ } else {+ ggml_vec_dot_q5_0_q8_0_default(n, s, vx, vy);+ }}--void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- const int qk = QK8_1;- const int nb = n / qk;-- assert(n % qk == 0);- assert(qk == QK5_1);-- const block_q5_1 * restrict x = vx;- const block_q8_1 * restrict y = vy;--#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- float summs0 = 0.0f;- float summs1 = 0.0f;-- uint32_t qh0;- uint32_t qh1;-- uint64_t tmp0[4];- uint64_t tmp1[4];-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q5_1 * restrict x0 = &x[i];- const block_q5_1 * restrict x1 = &x[i + 1];- const block_q8_1 * restrict y0 = &y[i];- const block_q8_1 * restrict y1 = &y[i + 1];-- const uint8x16_t m4b = vdupq_n_u8(0x0F);-- summs0 += GGML_FP16_TO_FP32(x0->m) * y0->s;- summs1 += GGML_FP16_TO_FP32(x1->m) * y1->s;-- // extract the 5th bit via lookup table ((b) << 4)- memcpy(&qh0, x0->qh, sizeof(qh0));- memcpy(&qh1, x1->qh, sizeof(qh1));-- tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];- tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];- tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];- tmp0[3] = table_b2b_0[(qh0 >> 24) ];-- tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];- tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];- tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];- tmp1[3] = table_b2b_0[(qh1 >> 24) ];-- const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));- const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));- const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));- const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));-- const uint8x16_t v0_0 = vld1q_u8(x0->qs);- const uint8x16_t v0_1 = vld1q_u8(x1->qs);-- // 4-bit -> 8-bit- const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));-- // add high bit- const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);- const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);- const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);- const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);-- // load y- const int8x16_t v1_0l = vld1q_s8(y0->qs);- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);- const int8x16_t v1_1l = vld1q_s8(y1->qs);- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);--#if defined(__ARM_FEATURE_DOTPROD)- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),- vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*y0->d);- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),- vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*y1->d);#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0lf), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0lf), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hf), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hf), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1lf), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1lf), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hf), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hf), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*y0->d);- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*y1->d);+#define static+#include "ggml_vec_dot_q5_0_q8_0.inc"+#undef static#endif- }-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;-#elif defined(__wasm_simd128__)- v128_t sumv = wasm_f32x4_splat(0.0f);-- float summs = 0.0f;-- uint32_t qh;- uint64_t tmp[4];-- // TODO: check if unrolling this is better- for (int i = 0; i < nb; ++i) {- const block_q5_1 * restrict x0 = &x[i];- const block_q8_1 * restrict y0 = &y[i];-- summs += GGML_FP16_TO_FP32(x0->m) * y0->s;-- const v128_t m4b = wasm_i8x16_splat(0x0F);- // extract the 5th bit- memcpy(&qh, x0->qh, sizeof(qh));-- tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];- tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];- tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];- tmp[3] = table_b2b_0[(qh >> 24) ];-- const v128_t qhl = wasm_v128_load(tmp + 0);- const v128_t qhh = wasm_v128_load(tmp + 2);-- const v128_t v0 = wasm_v128_load(x0->qs);-- // 4-bit -> 8-bit- const v128_t v0l = wasm_v128_and (v0, m4b);- const v128_t v0h = wasm_u8x16_shr(v0, 4);-- // add high bit- const v128_t v0lf = wasm_v128_or(v0l, qhl);- const v128_t v0hf = wasm_v128_or(v0h, qhh);-- // load y- const v128_t v1l = wasm_v128_load(y0->qs);- const v128_t v1h = wasm_v128_load(y0->qs + 16);-- // int8x16 -> int16x8- const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);- const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);- const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);- const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);-- const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);- const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);- const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);- const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);-- // dot product- sumv = wasm_f32x4_add(sumv,- wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),- wasm_i32x4_dot_i16x8(v0lfh, v1lh)),- wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),- wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),- wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * y0->d)));- }-- *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +- wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;-#elif defined(__AVX2__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- float summs = 0.0f;-- // Main loop- for (int i = 0; i < nb; i++) {- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));-- summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;-- __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);- __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);- bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));- bx = _mm256_or_si256(bx, bxhi);-- const __m256 dy = _mm256_set1_ps(y[i].d);- const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_us8_pairs_float_avx2(bx, by);-- acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);- }-- *s = hsum_float_8(acc) + summs;-#elif defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();- __m128i mask = _mm_set1_epi8(0x10);-- float summs = 0.0f;-- // Main loop- for (int i = 0; i < nb; i++) {- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));-- summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;-- __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);- const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);- __m128i bxhil = _mm256_castsi256_si128(bxhi);- __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);- bxhil = _mm_and_si128(bxhil, mask);- bxhih = _mm_and_si128(bxhih, mask);- __m128i bxl = _mm256_castsi256_si128(bx);- __m128i bxh = _mm256_extractf128_si256(bx, 1);- bxl = _mm_or_si128(bxl, bxhil);- bxh = _mm_or_si128(bxh, bxhih);- bx = MM256_SET_M128I(bxh, bxl);-- const __m256 dy = _mm256_set1_ps(y[i].d);- const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_us8_pairs_float_avx(bx, by);-- acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);- }-- *s = hsum_float_8(acc) + summs;-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;-- uint32_t qh;-- size_t vl = __riscv_vsetvl_e8m1(qk/2);-- // temporary registers for shift operations- vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);- vuint32m2_t vt_2 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);-- for (int i = 0; i < nb; i++) {- memcpy(&qh, x[i].qh, sizeof(uint32_t));-- // load qh- vuint32m2_t vqh = __riscv_vmv_v_x_u32m2(qh, vl);-- // ((qh >> (j + 0)) << 4) & 0x10;- vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(vqh, vt_1, vl);- vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);- vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(xhl_0, 0x10, vl);-- // ((qh >> (j + 12)) ) & 0x10;- vuint32m2_t xhr_1 = __riscv_vsrl_vv_u32m2(vqh, vt_2, vl);- vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(xhr_1, 0x10, vl);-- // narrowing- vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xha_0, vl);- vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);-- vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xha_1, vl);- vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);-- // load- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);-- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);-- vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);- vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);-- vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);- vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);-- vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);-- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);-- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);-- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);-- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);-- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;+#ifdef __x86_64__+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_default+#include "ggml_vec_dot_q5_1_q8_1.inc"+#undef ggml_vec_dot_q5_1_q8_1+#define ggml_vec_dot_q5_1_q8_1 __attribute__((__target__("avx"))) ggml_vec_dot_q5_1_q8_1_avx+#ifndef __AVX__+#define __UNDEF_AVX__+#define __AVX__+#endif+#include "ggml_vec_dot_q5_1_q8_1.inc"+#ifdef __UNDEF_AVX__+#undef __UNDEF_AVX__+#undef __AVX__+#endif+#undef ggml_vec_dot_q5_1_q8_1+#define ggml_vec_dot_q5_1_q8_1 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q5_1_q8_1_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__+#endif+#include "ggml_vec_dot_q5_1_q8_1.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__+#endif+#undef ggml_vec_dot_q5_1_q8_1+void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {+ ggml_vec_dot_q5_1_q8_1_avx2(n, s, vx, vy);+ } else if (X86_HAVE(AVX)) {+ ggml_vec_dot_q5_1_q8_1_avx(n, s, vx, vy);+ } else {+ ggml_vec_dot_q5_1_q8_1_default(n, s, vx, vy);}-- *s = sumf;+}#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- uint32_t qh;- memcpy(&qh, x[i].qh, sizeof(qh));-- int sumi = 0;-- for (int j = 0; j < qk/2; ++j) {- const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;- const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;-- const int32_t x0 = (x[i].qs[j] & 0xF) | xh_0;- const int32_t x1 = (x[i].qs[j] >> 4) | xh_1;-- sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);- }-- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;- }-- *s = sumf;+#define static+#include "ggml_vec_dot_q5_1_q8_1.inc"+#undef static#endif-}#ifdef __x86_64__-#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_default#include "ggml_vec_dot_q8_0_q8_0.inc"#undef ggml_vec_dot_q8_0_q8_0-#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q8_0_q8_0_avx#ifndef __AVX__#define __UNDEF_AVX__@@ -3200,7 +2638,6 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restri#undef __AVX__#endif#undef ggml_vec_dot_q8_0_q8_0-#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q8_0_q8_0_avx2#ifndef __AVX2__#define __UNDEF_AVX2__@@ -3212,7 +2649,6 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q8_0_q8_0-void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {ggml_vec_dot_q8_0_q8_0_avx2(n, s, vx, vy);@@ -3222,22 +2658,18 @@ void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restriggml_vec_dot_q8_0_q8_0_default(n, s, vx, vy);}}-#else#define static#include "ggml_vec_dot_q8_0_q8_0.inc"#undef static#endif-#if QK_K == 256#ifdef __x86_64__-#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_default#include "ggml_vec_dot_q2_K_q8_K_256.inc"#undef ggml_vec_dot_q2_K_q8_K-#define ggml_vec_dot_q2_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q2_K_q8_K_avx#ifndef __AVX__#define __UNDEF_AVX__@@ -3249,7 +2681,6 @@ void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restri#undef __AVX__#endif#undef ggml_vec_dot_q2_K_q8_K-#define ggml_vec_dot_q2_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q2_K_q8_K_avx2#ifndef __AVX2__#define __UNDEF_AVX2__@@ -3261,9 +2692,8 @@ void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q2_K_q8_K-void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {ggml_vec_dot_q2_K_q8_K_avx2(n, s, vx, vy);} else if (X86_HAVE(AVX)) {ggml_vec_dot_q2_K_q8_K_avx(n, s, vx, vy);@@ -3271,12 +2701,11 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restriggml_vec_dot_q2_K_q8_K_default(n, s, vx, vy);}}-#else#define static#include "ggml_vec_dot_q2_K_q8_K_256.inc"#undef static-#endif /* __x86_64__ */+#endif#else@@ -3294,7 +2723,7 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restriconst int32x4_t vzero = vdupq_n_s32(0);#endif- int8x16x4_t q2bytes;+ ggml_int8x16x4_t q2bytes;uint32_t aux32[2];const uint8_t * scales = (const uint8_t *)aux32;@@ -3319,7 +2748,7 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restriconst uint8x16_t q2bits = vld1q_u8(q2);- const int8x16x4_t q8bytes = vld1q_s8_x4(q8);+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits, m3));q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 2), m3));@@ -3558,11 +2987,9 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restri#if QK_K == 256#ifdef __x86_64__-#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_default#include "ggml_vec_dot_q3_K_q8_K_256.inc"#undef ggml_vec_dot_q3_K_q8_K-#define ggml_vec_dot_q3_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q3_K_q8_K_avx#ifndef __AVX__#define __UNDEF_AVX__@@ -3574,7 +3001,6 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX__#endif#undef ggml_vec_dot_q3_K_q8_K-#define ggml_vec_dot_q3_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q3_K_q8_K_avx2#ifndef __AVX2__#define __UNDEF_AVX2__@@ -3586,7 +3012,6 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q3_K_q8_K-void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {ggml_vec_dot_q3_K_q8_K_avx2(n, s, vx, vy);@@ -3596,7 +3021,6 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restriggml_vec_dot_q3_K_q8_K_default(n, s, vx, vy);}}-#else#define static#include "ggml_vec_dot_q3_K_q8_K_256.inc"@@ -3622,7 +3046,7 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restriconst uint8x16_t m3b = vdupq_n_u8(0x3);const uint8x16_t mh = vdupq_n_u8(4);- int8x16x4_t q3bytes;+ ggml_int8x16x4_t q3bytes;uint16_t aux16[2];int8_t * scales = (int8_t *)aux16;@@ -3631,11 +3055,11 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restrifor (int i = 0; i < nb; ++i) {- uint8x16x4_t q3h;+ ggml_uint8x16x4_t q3h;const uint8x8_t hbits = vld1_u8(x[i].hmask);const uint8x16_t q3bits = vld1q_u8(x[i].qs);- const int8x16x4_t q8bytes = vld1q_s8_x4(y[i].qs);+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(y[i].qs);const uint16_t a = *(const uint16_t *)x[i].scales;aux16[0] = a & 0x0f0f;@@ -3965,11 +3389,9 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri#if QK_K == 256#ifdef __x86_64__-#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_default#include "ggml_vec_dot_q4_K_q8_K_256.inc"#undef ggml_vec_dot_q4_K_q8_K-#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q4_K_q8_K_avx#ifndef __AVX__#define __UNDEF_AVX__@@ -3981,7 +3403,6 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX__#endif#undef ggml_vec_dot_q4_K_q8_K-#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_K_q8_K_avx2#ifndef __AVX2__#define __UNDEF_AVX2__@@ -3993,7 +3414,6 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q4_K_q8_K-void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {ggml_vec_dot_q4_K_q8_K_avx2(n, s, vx, vy);@@ -4003,7 +3423,6 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restriggml_vec_dot_q4_K_q8_K_default(n, s, vx, vy);}}-#else#define static#include "ggml_vec_dot_q4_K_q8_K_256.inc"@@ -4029,8 +3448,8 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrifloat sumf = 0;- int8x16x2_t q4bytes;- int8x16x4_t q8bytes;+ ggml_int8x16x2_t q4bytes;+ ggml_int8x16x4_t q8bytes;float sum_mins = 0.f;@@ -4051,10 +3470,10 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restriconst float d = y[i].d * (float)x[i].d[0];- const uint8x16x2_t q4bits = vld1q_u8_x2(q4);+ const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4);#ifdef __ARM_FEATURE_DOTPROD- q8bytes = vld1q_s8_x4(q8);+ q8bytes = ggml_vld1q_s8_x4(q8);q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));@@ -4068,7 +3487,7 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restriconst int32_t sumi2 = vaddvq_s32(p2) * scales[1];#else- q8bytes = vld1q_s8_x4(q8);+ q8bytes = ggml_vld1q_s8_x4(q8);q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),@@ -4280,11 +3699,9 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restri#if QK_K == 256#ifdef __x86_64__-#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_default#include "ggml_vec_dot_q5_K_q8_K_256.inc"#undef ggml_vec_dot_q5_K_q8_K-#define ggml_vec_dot_q5_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q5_K_q8_K_avx#ifndef __AVX__#define __UNDEF_AVX__@@ -4296,7 +3713,6 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX__#endif#undef ggml_vec_dot_q5_K_q8_K-#define ggml_vec_dot_q5_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q5_K_q8_K_avx2#ifndef __AVX2__#define __UNDEF_AVX2__@@ -4308,7 +3724,6 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q5_K_q8_K-void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {ggml_vec_dot_q5_K_q8_K_avx2(n, s, vx, vy);@@ -4318,7 +3733,6 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restriggml_vec_dot_q5_K_q8_K_default(n, s, vx, vy);}}-#else#define static#include "ggml_vec_dot_q5_K_q8_K_256.inc"@@ -4343,8 +3757,8 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restriconst int32x4_t mzero = vdupq_n_s32(0);#endif- int8x16x4_t q5bytes;- uint8x16x4_t q5h;+ ggml_int8x16x4_t q5bytes;+ ggml_uint8x16x4_t q5h;float sumf = 0;@@ -4359,8 +3773,8 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restriconst uint8x8_t qhbits = vld1_u8(qh);- const uint8x16x2_t q5bits = vld1q_u8_x2(q5);- const int8x16x4_t q8bytes = vld1q_s8_x4(q8);+ const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5);+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);const uint8x16_t htmp = vcombine_u8(qhbits, vshr_n_u8(qhbits, 1));q5h.val[0] = vbicq_u8(mh, vshlq_n_u8(htmp, 4));@@ -4618,11 +4032,9 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri#if QK_K == 256#ifdef __x86_64__-#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_default#include "ggml_vec_dot_q6_K_q8_K_256.inc"#undef ggml_vec_dot_q6_K_q8_K-#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q6_K_q8_K_avx#ifndef __AVX__#define __UNDEF_AVX__@@ -4634,7 +4046,6 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX__#endif#undef ggml_vec_dot_q6_K_q8_K-#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q6_K_q8_K_avx2#ifndef __AVX2__#define __UNDEF_AVX2__@@ -4646,9 +4057,8 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q6_K_q8_K-void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {ggml_vec_dot_q6_K_q8_K_avx2(n, s, vx, vy);} else if (X86_HAVE(AVX)) {ggml_vec_dot_q6_K_q8_K_avx(n, s, vx, vy);@@ -4656,7 +4066,6 @@ void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restriggml_vec_dot_q6_K_q8_K_default(n, s, vx, vy);}}-#else#define static#include "ggml_vec_dot_q6_K_q8_K_256.inc"@@ -4685,8 +4094,8 @@ void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restriconst uint8x16_t mone = vdupq_n_u8(3);- int8x16x4_t q6bytes;- uint8x16x4_t q6h;+ ggml_int8x16x4_t q6bytes;+ ggml_uint8x16x4_t q6h;for (int i = 0; i < nb; ++i) {@@ -4700,9 +4109,9 @@ void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restriint32_t isum = 0;- uint8x16_t qhbits = vld1q_u8(qh);- uint8x16x2_t q6bits = vld1q_u8_x2(q6);- int8x16x4_t q8bytes = vld1q_s8_x4(q8);+ uint8x16_t qhbits = vld1q_u8(qh);+ ggml_uint8x16x2_t q6bits = ggml_vld1q_u8_x2(q6);+ ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8);q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits), 4);uint8x16_t shifted = vshrq_n_u8(qhbits, 2);diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 138daf86..d5840487 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -1,33 +1,60 @@// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi-#include "ggml.h"++asm(".ident\t\"\\n\\n\+llama.cpp (MIT License)\\n\+Copyright (c) 2023 Georgi Gerganov\\n\+Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\\n\+Copyright (c) 2023 Yuji Hirose\\n\+Copyright (c) 2022 Niels Lohmann <https://nlohmann.me>\\n\+Copyright (c) 2022 Bjoern Hoehrmann <[email protected]>\\n\+Copyright (c) 2017 Sean Barrett\\n\+\\n\+Permission is hereby granted, free of charge, to any person obtaining a copy\\n\+of this software and associated documentation files (the \\\"Software\\\"), to deal\\n\+in the Software without restriction, including without limitation the rights\\n\+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\\n\+copies of the Software, and to permit persons to whom the Software is\\n\+furnished to do so, subject to the following conditions:\\n\+\\n\+The above copyright notice and this permission notice shall be included in all\\n\+copies or substantial portions of the Software.\\n\+\\n\+THE SOFTWARE IS PROVIDED \\\"AS IS\\\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\\n\+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\\n\+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\\n\+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\\n\+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\\n\+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\\n\+SOFTWARE.\"");+#include "ggml-impl.h"#include "ggml-quants.h"-#include "ggml-cuda.h"#include "ggml-metal.h"+#include "ggml-cuda.h"+#include "libc/log/backtrace.internal.h"+#include <alloca.h>#include <assert.h>-#include <cosmo.h>#include <errno.h>-#include <fcntl.h>-#include <float.h>+#include <time.h>+#include <math.h>+#include <stdlib.h>+#include <string.h>+#include <stdint.h>#include <inttypes.h>+#include <stdio.h>+#include <float.h>#include <limits.h>-#include <math.h>-#include <pthread.h>-#include <sched.h>-#include <signal.h>#include <stdarg.h>+#include <signal.h>+#include <unistd.h>+#include <cosmo.h>++#include <pthread.h>#include <stdatomic.h>-#include <stdint.h>-#include <stdio.h>-#include <stdlib.h>-#include <string.h>-#include <sys/auxv.h>-#include <sys/mman.h>-#include <sys/stat.h>#include <sys/types.h>-#include <time.h>+#include <sys/stat.h>#include <unistd.h>typedef void * thread_ret_t;@@ -36,6 +63,53 @@ typedef void * thread_ret_t;#include <hbwmalloc.h>#endif+#if defined(__APPLE__)+#include <TargetConditionals.h>+#endif++#if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)) && \+ (!defined(TARGET_OS_TV) && !defined(TARGET_OS_WATCH))++#include <sys/wait.h>++void ggml_print_backtrace(void) {+ /*+ #include <execinfo.h>+ #include <dlfcn.h>++ void * trace[100];++ int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));++ backtrace_symbols_fd(trace, nptrs, STDERR_FILENO);+ */++ // backtrack_symbols does not show line numbers, use gdb instead+ char attach[32];+ snprintf(attach, sizeof(attach), "attach %d", getpid());+ int pid = fork();+ if (pid == 0) {+ execlp("gdb", "gdb", "--batch",+ "-ex", "set style enabled on",+ "-ex", attach,+ "-ex", "bt -frame-info source-and-location",+ "-ex", "detach",+ "-ex", "quit",+ NULL);+ } else {+ waitpid(pid, NULL, 0);+ }+}+#elif defined(__COSMOPOLITAN__)+void ggml_print_backtrace(void) {+ ShowBacktrace(2, __builtin_frame_address(0));+}+#else+void ggml_print_backtrace(void) {+ // platform not supported+}+#endif+/*#define GGML_PERF*/#define GGML_DEBUG 0#define GGML_GELU_FP16@@ -72,33 +146,6 @@ typedef void * thread_ret_t;#define GGML_PRINT(...) printf(__VA_ARGS__)-asm(".ident\t\"\\n\\n\-llama.cpp (MIT License)\\n\-Copyright (c) 2023 Georgi Gerganov\\n\-Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\\n\-Copyright (c) 2023 Yuji Hirose\\n\-Copyright (c) 2022 Niels Lohmann <https://nlohmann.me>\\n\-Copyright (c) 2022 Bjoern Hoehrmann <[email protected]>\\n\-Copyright (c) 2017 Sean Barrett\\n\-\\n\-Permission is hereby granted, free of charge, to any person obtaining a copy\\n\-of this software and associated documentation files (the \\\"Software\\\"), to deal\\n\-in the Software without restriction, including without limitation the rights\\n\-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\\n\-copies of the Software, and to permit persons to whom the Software is\\n\-furnished to do so, subject to the following conditions:\\n\-\\n\-The above copyright notice and this permission notice shall be included in all\\n\-copies or substantial portions of the Software.\\n\-\\n\-THE SOFTWARE IS PROVIDED \\\"AS IS\\\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\\n\-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\\n\-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\\n\-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\\n\-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\\n\-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\\n\-SOFTWARE.\"");-//// end of logging block//@@ -109,48 +156,82 @@ void ggml_interrupt(bool state) {is_interrupted = state;}-static void * ggml_aligned_malloc(size_t size) {+#ifdef GGML_USE_ACCELERATE+// uncomment to use vDSP for soft max computation+// note: not sure if it is actually faster+//#define GGML_SOFT_MAX_ACCELERATE+#endif++#if defined(_MSC_VER) || defined(__MINGW32__)+#define GGML_ALIGNED_MALLOC(size) _aligned_malloc(size, GGML_MEM_ALIGN)+#define GGML_ALIGNED_FREE(ptr) _aligned_free(ptr)+#else+inline static void * ggml_aligned_malloc(size_t size) {if (size == 0) {GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");return NULL;}void * aligned_memory = NULL;- errno_t err = posix_memalign(&aligned_memory, getauxval(AT_PAGESZ), size);- if (err) {- GGML_PRINT("%s: %s (attempted to allocate %6.2f MB)\n", __func__, strerror(err), size/(1024.0*1024.0));+#ifdef GGML_USE_CPU_HBM+ int result = hbw_posix_memalign(&aligned_memory, 16, size);+#elif GGML_USE_METAL+ int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);+#else+ int result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);+#endif+ if (result != 0) {+ // Handle allocation failure+ const char *error_desc = "unknown allocation error";+ switch (result) {+ case EINVAL:+ error_desc = "invalid alignment value";+ break;+ case ENOMEM:+ error_desc = "insufficient memory";+ break;+ }+ GGML_PRINT("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));return NULL;}return aligned_memory;}--static void ggml_aligned_free(void *p) {- free(p);-}+#define GGML_ALIGNED_MALLOC(size) ggml_aligned_malloc(size)+#ifdef GGML_USE_CPU_HBM+#define GGML_ALIGNED_FREE(ptr) if(NULL != ptr) hbw_free(ptr)+#else+#define GGML_ALIGNED_FREE(ptr) free(ptr)+#endif+#endif#define UNUSED GGML_UNUSED#define SWAP(x, y, T) do { T SWAP = x; x = y; y = SWAP; } while (0)-//-// tensor access macros-//--#define GGML_TENSOR_UNARY_OP_LOCALS \- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)--#define GGML_TENSOR_BINARY_OP_LOCALS \- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \- GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \- GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)+#if defined(GGML_USE_ACCELERATE)+#include <Accelerate/Accelerate.h>+#if defined(GGML_USE_CLBLAST) // allow usage of CLBlast alongside Accelerate functions+#include "ggml-opencl.h"+#endif+#elif defined(GGML_USE_OPENBLAS)+#if defined(GGML_BLAS_USE_MKL)+#include <mkl.h>+#else+#include <cblas.h>+#endif+#elif defined(GGML_USE_CUBLAS)+#include "ggml-cuda.h"+#elif defined(GGML_USE_CLBLAST)+#include "ggml-opencl.h"+#endif// floating point type used to accumulate sumstypedef double ggml_float;+#undef MIN+#undef MAX++#define MIN(a, b) ((a) < (b) ? (a) : (b))+#define MAX(a, b) ((a) > (b) ? (a) : (b))+//// global data//@@ -471,6 +552,18 @@ ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type) {// simd mappings//+#if defined(__ARM_NEON)+#if !defined(__aarch64__)++// 64-bit compatibility++inline static float vaddvq_f32(float32x4_t v) {+ return vgetq_lane_f32(v, 0) + vgetq_lane_f32(v, 1) + vgetq_lane_f32(v, 2) + vgetq_lane_f32(v, 3);+}++#endif+#endif+// we define a common set of C macros which map to specific intrinsics based on the current architecture// we then implement the fundamental computation operations below using only these macros// adding support for new architectures requires to define the corresponding SIMD macros@@ -1358,6 +1451,7 @@ inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) {inline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }inline static void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expf(x[i])-1; }inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }+inline static void ggml_vec_leaky_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.1f*x[i]; }static const float GELU_COEF_A = 0.044715f;static const float GELU_QUICK_COEF = -1.702f;@@ -1557,6 +1651,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"GROUP_NORM","MUL_MAT",+ "MUL_MAT_ID","OUT_PROD","SCALE",@@ -1578,17 +1673,13 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"ROPE_BACK","ALIBI","CLAMP",- "CONV_1D",- "CONV_1D_STAGE_0",- "CONV_1D_STAGE_1","CONV_TRANSPOSE_1D",- "CONV_2D",- "CONV_2D_STAGE_0",- "CONV_2D_STAGE_1",+ "IM2COL","CONV_TRANSPOSE_2D","POOL_1D","POOL_2D","UPSCALE",+ "ARGSORT","FLASH_ATTN","FLASH_FF",@@ -1615,7 +1706,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"CROSS_ENTROPY_LOSS_BACK",};-static_assert(GGML_OP_COUNT == 73, "GGML_OP_COUNT != 73");+static_assert(GGML_OP_COUNT == 70, "GGML_OP_COUNT != 70");static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"none",@@ -1644,6 +1735,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"group_norm(x)","X*Y",+ "X[i]*Y","X*Y","x*v",@@ -1665,17 +1757,13 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"rope_back(x)","alibi(x)","clamp(x)",- "conv_1d(x)",- "conv_1d_stage_0(x)",- "conv_1d_stage_1(x)","conv_transpose_1d(x)",- "conv_2d(x)",- "conv_2d_stage_0(x)",- "conv_2d_stage_1(x)",+ "im2col(x)","conv_transpose_2d(x)","pool_1d(x)","pool_2d(x)","upscale(x)",+ "argsort(x)","flash_attn(x)","flash_ff(x)",@@ -1702,10 +1790,28 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"cross_entropy_loss_back(x,y)",};-static_assert(GGML_OP_COUNT == 73, "GGML_OP_COUNT != 73");+static_assert(GGML_OP_COUNT == 70, "GGML_OP_COUNT != 70");static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");++static const char * GGML_UNARY_OP_NAME[GGML_UNARY_OP_COUNT] = {+ "ABS",+ "SGN",+ "NEG",+ "STEP",+ "TANH",+ "ELU",+ "RELU",+ "GELU",+ "GELU_QUICK",+ "SILU",+ "LEAKY",+};++static_assert(GGML_UNARY_OP_COUNT == 11, "GGML_UNARY_OP_COUNT != 11");++static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");static_assert(sizeof(struct ggml_tensor)%GGML_MEM_ALIGN == 0, "ggml_tensor size must be a multiple of GGML_MEM_ALIGN");@@ -1725,18 +1831,13 @@ static void ggml_setup_op_has_task_pass(void) {p[GGML_OP_ACC ] = true;p[GGML_OP_MUL_MAT ] = true;+ p[GGML_OP_MUL_MAT_ID ] = true;p[GGML_OP_OUT_PROD ] = true;p[GGML_OP_SET ] = true;p[GGML_OP_GET_ROWS_BACK ] = true;p[GGML_OP_DIAG_MASK_INF ] = true;p[GGML_OP_DIAG_MASK_ZERO ] = true;- p[GGML_OP_CONV_1D ] = true;- p[GGML_OP_CONV_1D_STAGE_0 ] = true;- p[GGML_OP_CONV_1D_STAGE_1 ] = true;p[GGML_OP_CONV_TRANSPOSE_1D ] = true;- p[GGML_OP_CONV_2D ] = true;- p[GGML_OP_CONV_2D_STAGE_0 ] = true;- p[GGML_OP_CONV_2D_STAGE_1 ] = true;p[GGML_OP_CONV_TRANSPOSE_2D ] = true;p[GGML_OP_FLASH_ATTN_BACK ] = true;p[GGML_OP_CROSS_ENTROPY_LOSS ] = true;@@ -1833,7 +1934,7 @@ void ggml_numa_init(void) {}if (!IsLinux()) {- return; // todo+ return;}struct stat st;@@ -1983,6 +2084,20 @@ const char * ggml_op_symbol(enum ggml_op op) {return GGML_OP_SYMBOL[op];}+const char * ggml_unary_op_name(enum ggml_unary_op op) {+ return GGML_UNARY_OP_NAME[op];+}++const char * ggml_op_desc(const struct ggml_tensor * t) {+ if (t->op == GGML_OP_UNARY) {+ enum ggml_unary_op uop = ggml_get_unary_op(t);+ return ggml_unary_op_name(uop);+ }+ else {+ return ggml_op_name(t->op);+ }+}+size_t ggml_element_size(const struct ggml_tensor * tensor) {return ggml_type_size(tensor->type);}@@ -2187,7 +2302,6 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {}ggml_init_cublas();-#if defined(GGML_USE_CLBLAST)ggml_cl_init();#endif@@ -2227,7 +2341,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {*ctx = (struct ggml_context) {/*.mem_size =*/ mem_size,- /*.mem_buffer =*/ params.mem_buffer ? params.mem_buffer : ggml_aligned_malloc(mem_size),+ /*.mem_buffer =*/ params.mem_buffer ? params.mem_buffer : GGML_ALIGNED_MALLOC(mem_size),/*.mem_buffer_owned =*/ params.mem_buffer ? false : true,/*.no_alloc =*/ params.no_alloc,/*.no_alloc_save =*/ params.no_alloc,@@ -2263,7 +2377,7 @@ void ggml_free(struct ggml_context * ctx) {__func__, i, ggml_used_mem(ctx));if (ctx->mem_buffer_owned) {- ggml_aligned_free(ctx->mem_buffer);+ GGML_ALIGNED_FREE(ctx->mem_buffer);}found = true;@@ -3114,9 +3228,7 @@ static struct ggml_tensor * ggml_add_impl(struct ggml_tensor * a,struct ggml_tensor * b,bool inplace) {- // TODO: support less-strict constraint- // GGML_ASSERT(ggml_can_repeat(b, a));- GGML_ASSERT(ggml_can_repeat_rows(b, a));+ GGML_ASSERT(ggml_can_repeat(b, a));bool is_node = false;@@ -3331,9 +3443,7 @@ static struct ggml_tensor * ggml_mul_impl(struct ggml_tensor * a,struct ggml_tensor * b,bool inplace) {- // TODO: support less-strict constraint- // GGML_ASSERT(ggml_can_repeat(b, a));- GGML_ASSERT(ggml_can_repeat_rows(b, a));+ GGML_ASSERT(ggml_can_repeat(b, a));bool is_node = false;@@ -3378,7 +3488,7 @@ static struct ggml_tensor * ggml_div_impl(struct ggml_tensor * a,struct ggml_tensor * b,bool inplace) {- GGML_ASSERT(ggml_are_same_shape(a, b));+ GGML_ASSERT(ggml_can_repeat(b, a));bool is_node = false;@@ -3775,6 +3885,14 @@ struct ggml_tensor * ggml_relu_inplace(return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_RELU);}+// ggml_leaky++struct ggml_tensor * ggml_leaky(+ struct ggml_context * ctx,+ struct ggml_tensor * a) {+ return ggml_unary(ctx, a, GGML_UNARY_OP_LEAKY);+}+// ggml_gelustruct ggml_tensor * ggml_gelu(@@ -4008,6 +4126,51 @@ struct ggml_tensor * ggml_mul_mat(return result;}+// ggml_mul_mat_id++struct ggml_tensor * ggml_mul_mat_id(+ struct ggml_context * ctx,+ struct ggml_tensor * const as[],+ int n_as,+ struct ggml_tensor * ids,+ int id,+ struct ggml_tensor * b) {++ GGML_ASSERT(ids->type == GGML_TYPE_I32);+ GGML_ASSERT(ids->ne[2] == 1 && ids->ne[3] == 1);+ GGML_ASSERT(ids->ne[1] == b->ne[1]);+ GGML_ASSERT(ids->ne[2] == b->ne[2] && ids->ne[3] == b->ne[3]);+ GGML_ASSERT(n_as > 0 && n_as <= GGML_MAX_SRC - 2);+ GGML_ASSERT(id >= 0 && id < ids->ne[0]);++ bool is_node = false;++ if (as[0]->grad || b->grad) {+ is_node = true;+ }++ const int64_t ne[4] = { as[0]->ne[1], b->ne[1], b->ne[2], b->ne[3] };+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, MAX(as[0]->n_dims, b->n_dims), ne);++ ggml_set_op_params_i32(result, 0, id);+ ggml_set_op_params_i32(result, 1, n_as);++ result->op = GGML_OP_MUL_MAT_ID;+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;+ result->src[0] = ids;+ result->src[1] = b;++ for (int i = 0; i < n_as; i++) {+ struct ggml_tensor * a = as[i];+ GGML_ASSERT(ggml_are_same_shape(as[0], a));+ GGML_ASSERT(ggml_can_mul_mat(a, b));+ GGML_ASSERT(!ggml_is_transposed(a));+ result->src[i + 2] = a;+ }++ return result;+}+// ggml_out_prodstruct ggml_tensor * ggml_out_prod(@@ -4161,7 +4324,7 @@ struct ggml_tensor * ggml_set_2d_inplace(struct ggml_tensor * b,size_t nb1,size_t offset) {- return ggml_set_impl(ctx, a, b, nb1, a->nb[2], a->nb[3], offset, false);+ return ggml_set_impl(ctx, a, b, nb1, a->nb[2], a->nb[3], offset, true);}// ggml_cpy@@ -4625,7 +4788,9 @@ struct ggml_tensor * ggml_get_rows(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b) {- GGML_ASSERT(ggml_is_matrix(a) && ggml_is_vector(b) && b->type == GGML_TYPE_I32);+ GGML_ASSERT(a->ne[2] == b->ne[1]);+ GGML_ASSERT(b->ne[3] == 1);+ GGML_ASSERT(b->type == GGML_TYPE_I32);bool is_node = false;@@ -4635,7 +4800,7 @@ struct ggml_tensor * ggml_get_rows(// TODO: implement non F32 return//struct ggml_tensor * result = ggml_new_tensor_2d(ctx, a->type, a->ne[0], b->ne[0]);- struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, a->ne[0], b->ne[0]);+ struct ggml_tensor * result = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, a->ne[0], b->ne[0], b->ne[1], b->ne[2]);result->op = GGML_OP_GET_ROWS;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -4778,7 +4943,17 @@ struct ggml_tensor * ggml_diag_mask_zero_inplace(static struct ggml_tensor * ggml_soft_max_impl(struct ggml_context * ctx,struct ggml_tensor * a,+ struct ggml_tensor * mask,+ float scale,bool inplace) {+ GGML_ASSERT(ggml_is_contiguous(a));+ if (mask) {+ GGML_ASSERT(ggml_is_contiguous(mask));+ GGML_ASSERT(mask->ne[2] == 1);+ GGML_ASSERT(mask->ne[3] == 1);+ GGML_ASSERT(ggml_can_repeat_rows(mask, a));+ }+bool is_node = false;if (a->grad) {@@ -4787,9 +4962,13 @@ static struct ggml_tensor * ggml_soft_max_impl(struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);+ float params[] = { scale };+ ggml_set_op_params(result, params, sizeof(params));+result->op = GGML_OP_SOFT_MAX;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;+ result->src[1] = mask;return result;}@@ -4797,13 +4976,21 @@ static struct ggml_tensor * ggml_soft_max_impl(struct ggml_tensor * ggml_soft_max(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_soft_max_impl(ctx, a, false);+ return ggml_soft_max_impl(ctx, a, NULL, 1.0f, false);}struct ggml_tensor * ggml_soft_max_inplace(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_soft_max_impl(ctx, a, true);+ return ggml_soft_max_impl(ctx, a, NULL, 1.0f, true);+}++struct ggml_tensor * ggml_soft_max_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * mask,+ float scale) {+ return ggml_soft_max_impl(ctx, a, mask, scale, false);}// ggml_soft_max_back@@ -5082,82 +5269,6 @@ static int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p,return (ins + 2 * p - d * (ks - 1) - 1) / s + 1;}-// im2col: [N, IC, IL] => [N, OL, IC*K]-// a: [OC,IC, K]-// b: [N, IC, IL]-// result: [N, OL, IC*K]-static struct ggml_tensor * ggml_conv_1d_stage_0(- struct ggml_context * ctx,- struct ggml_tensor * a,- struct ggml_tensor * b,- int s0,- int p0,- int d0) {- GGML_ASSERT(a->ne[1] == b->ne[1]);- bool is_node = false;-- if (a->grad || b->grad) {- GGML_ASSERT(false); // TODO: implement backward- is_node = true;- }-- const int64_t OL = ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0);-- const int64_t ne[4] = {- a->ne[1] * a->ne[0],- OL,- b->ne[2],- 1,- };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F16, 4, ne);-- int32_t params[] = { s0, p0, d0 };- ggml_set_op_params(result, params, sizeof(params));-- result->op = GGML_OP_CONV_1D_STAGE_0;- result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;- result->src[0] = a;- result->src[1] = b;-- return result;-}--// ggml_conv_1d_stage_1--// gemm: [N, OC, OL] = [OC, IC * K] x [N*OL, IC * K]-// a: [OC, IC, K]-// b: [N, OL, IC * K]-// result: [N, OC, OL]-static struct ggml_tensor * ggml_conv_1d_stage_1(- struct ggml_context * ctx,- struct ggml_tensor * a,- struct ggml_tensor * b) {-- bool is_node = false;-- if (a->grad || b->grad) {- GGML_ASSERT(false); // TODO: implement backward- is_node = true;- }-- const int64_t ne[4] = {- b->ne[1],- a->ne[2],- b->ne[2],- 1,- };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);-- result->op = GGML_OP_CONV_1D_STAGE_1;- result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;- result->src[0] = a;- result->src[1] = b;-- return result;-}--// ggml_conv_1d-GGML_API struct ggml_tensor * ggml_conv_1d(struct ggml_context * ctx,struct ggml_tensor * a,@@ -5165,43 +5276,17 @@ GGML_API struct ggml_tensor * ggml_conv_1d(int s0,int p0,int d0) {- struct ggml_tensor * result = ggml_conv_1d_stage_0(ctx, a, b, s0, p0, d0);- result = ggml_conv_1d_stage_1(ctx, a, result);- return result;-}+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false); // [N, OL, IC * K]-// GGML_API struct ggml_tensor * ggml_conv_1d(-// struct ggml_context * ctx,-// struct ggml_tensor * a,-// struct ggml_tensor * b,-// int s0,-// int p0,-// int d0) {-// GGML_ASSERT(ggml_is_matrix(b));-// GGML_ASSERT(a->ne[1] == b->ne[1]);-// bool is_node = false;+ struct ggml_tensor * result =+ ggml_mul_mat(ctx,+ ggml_reshape_2d(ctx, im2col, im2col->ne[0], (im2col->ne[2] * im2col->ne[1])), // [N, OL, IC * K] => [N*OL, IC * K]+ ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1]), a->ne[2])); // [OC,IC, K] => [OC, IC * K]-// if (a->grad || b->grad) {-// GGML_ASSERT(false); // TODO: implement backward-// is_node = true;-// }+ result = ggml_reshape_3d(ctx, result, im2col->ne[1], a->ne[2], im2col->ne[2]); // [N, OC, OL]-// const int64_t ne[4] = {-// ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0),-// a->ne[2], 1, 1,-// };-// struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 2, ne);--// int32_t params[] = { s0, p0, d0 };-// ggml_set_op_params(result, params, sizeof(params));--// result->op = GGML_OP_CONV_1D;-// result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;-// result->src[0] = a;-// result->src[1] = b;--// return result;-// }+ return result;+}// ggml_conv_1d_ph@@ -5264,7 +5349,7 @@ GGML_API struct ggml_tensor * ggml_conv_transpose_1d(// a: [OC,IC, KH, KW]// b: [N, IC, IH, IW]// result: [N, OH, OW, IC*KH*KW]-static struct ggml_tensor * ggml_conv_2d_stage_0(+struct ggml_tensor * ggml_im2col(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,@@ -5273,9 +5358,14 @@ static struct ggml_tensor * ggml_conv_2d_stage_0(int p0,int p1,int d0,- int d1) {+ int d1,+ bool is_2D) {- GGML_ASSERT(a->ne[2] == b->ne[2]);+ if(is_2D) {+ GGML_ASSERT(a->ne[2] == b->ne[2]);+ } else {+ GGML_ASSERT(a->ne[1] == b->ne[1]);+ }bool is_node = false;if (a->grad || b->grad) {@@ -5283,81 +5373,51 @@ static struct ggml_tensor * ggml_conv_2d_stage_0(is_node = true;}- const int64_t OH = ggml_calc_conv_output_size(b->ne[1], a->ne[1], s1, p1, d1);- const int64_t OW = ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0);+ const int64_t OH = is_2D ? ggml_calc_conv_output_size(b->ne[1], a->ne[1], s1, p1, d1) : 0;+ const int64_t OW = ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0);const int64_t ne[4] = {- a->ne[2] * a->ne[1] * a->ne[0],+ is_2D ? (a->ne[2] * a->ne[1] * a->ne[0]) : a->ne[1] * a->ne[0],OW,- OH,- b->ne[3],+ is_2D ? OH : b->ne[2],+ is_2D ? b->ne[3] : 1,};- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F16, 4, ne);- int32_t params[] = { s0, s1, p0, p1, d0, d1 };+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F16, 4, ne);+ int32_t params[] = { s0, s1, p0, p1, d0, d1, (is_2D ? 1 : 0) };ggml_set_op_params(result, params, sizeof(params));- result->op = GGML_OP_CONV_2D_STAGE_0;- result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;- result->src[0] = a;- result->src[1] = b;-- return result;--}--// gemm: [N, OC, OH, OW] = [OC, IC * KH * KW] x [N*OH*OW, IC * KH * KW]-// a: [OC, IC, KH, KW]-// b: [N, OH, OW, IC * KH * KW]-// result: [N, OC, OH, OW]-static struct ggml_tensor * ggml_conv_2d_stage_1(- struct ggml_context * ctx,- struct ggml_tensor * a,- struct ggml_tensor * b) {-- bool is_node = false;-- if (a->grad || b->grad) {- GGML_ASSERT(false); // TODO: implement backward- is_node = true;- }-- const int64_t ne[4] = {- b->ne[1],- b->ne[2],- a->ne[3],- b->ne[3],- };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);-- result->op = GGML_OP_CONV_2D_STAGE_1;+ result->op = GGML_OP_IM2COL;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;result->src[1] = b;return result;-}// a: [OC,IC, KH, KW]// b: [N, IC, IH, IW]// result: [N, OC, OH, OW]struct ggml_tensor * ggml_conv_2d(- struct ggml_context * ctx,- struct ggml_tensor * a,- struct ggml_tensor * b,- int s0,- int s1,- int p0,- int p1,- int d0,- int d1) {+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int s0,+ int s1,+ int p0,+ int p1,+ int d0,+ int d1) {+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true); // [N, OH, OW, IC * KH * KW]- struct ggml_tensor * result = ggml_conv_2d_stage_0(ctx, a, b, s0, s1, p0, p1, d0, d1); // [N, OH, OW, IC * KH * KW]- result = ggml_conv_2d_stage_1(ctx, a, result);+ struct ggml_tensor * result =+ ggml_mul_mat(ctx,+ ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]), // [N, OH, OW, IC * KH * KW] => [N*OH*OW, IC * KH * KW]+ ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1] * a->ne[2]), a->ne[3])); // [OC,IC, KH, KW] => [OC, IC * KH * KW]- return result;+ result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], a->ne[3], im2col->ne[3]); // [N, OC, OH, OW]+ return result;}// ggml_conv_2d_sk_p0@@ -5417,7 +5477,7 @@ struct ggml_tensor * ggml_conv_transpose_2d_p0(// ggml_pool_*-static int64_t ggml_calc_pool_output_size(int64_t ins, int ks, int s, int p) {+static int64_t ggml_calc_pool_output_size(int64_t ins, int ks, int s, float p) {return (ins + 2 * p - ks) / s + 1;}@@ -5464,8 +5524,8 @@ struct ggml_tensor * ggml_pool_2d(int k1,int s0,int s1,- int p0,- int p1) {+ float p0,+ float p1) {bool is_node = false;@@ -5525,6 +5585,43 @@ struct ggml_tensor * ggml_upscale(return ggml_upscale_impl(ctx, a, scale_factor);}+// ggml_argsort++struct ggml_tensor * ggml_argsort(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_sort_order order) {+ bool is_node = false;++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_I32, a->n_dims, a->ne);++ ggml_set_op_params_i32(result, 0, (int32_t) order);++ result->op = GGML_OP_ARGSORT;+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;+ result->src[0] = a;++ return result;+}++// ggml_top_k++struct ggml_tensor * ggml_top_k(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int k) {+ GGML_ASSERT(a->ne[0] >= k);++ struct ggml_tensor * result = ggml_argsort(ctx, a, GGML_SORT_DESC);++ result = ggml_view_4d(ctx, result,+ k, result->ne[1], result->ne[2], result->ne[3],+ result->nb[1], result->nb[2], result->nb[3],+ 0);++ return result;+}+// ggml_flash_attnstruct ggml_tensor * ggml_flash_attn(@@ -6884,7 +6981,7 @@ static void ggml_compute_forward_add_f32(const struct ggml_tensor * src0,const struct ggml_tensor * src1,struct ggml_tensor * dst) {- GGML_ASSERT(ggml_can_repeat_rows(src1, src0) && ggml_are_same_shape(src0, dst));+ GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;@@ -6917,16 +7014,19 @@ static void ggml_compute_forward_add_f32(const int64_t i13 = i03 % ne13;const int64_t i12 = i02 % ne12;const int64_t i11 = i01 % ne11;+ const int64_t nr0 = ne00 / ne10;float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);+ for (int64_t r = 0; r < nr0; ++r) {#ifdef GGML_USE_ACCELERATE- vDSP_vadd(src0_ptr, 1, src1_ptr, 1, dst_ptr, 1, ne00);+ vDSP_vadd(src0_ptr + r*ne10, 1, src1_ptr, 1, dst_ptr + r*ne10, 1, ne10);#else- ggml_vec_add_f32(ne00, dst_ptr, src0_ptr, src1_ptr);+ ggml_vec_add_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);#endif+ }}} else {// src1 is not contiguous@@ -6943,8 +7043,9 @@ static void ggml_compute_forward_add_f32(float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);- for (int i0 = 0; i0 < ne0; i0++) {- float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i0*nb10);+ for (int64_t i0 = 0; i0 < ne0; ++i0) {+ const int64_t i10 = i0 % ne10;+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);dst_ptr[i0] = src0_ptr[i0] + *src1_ptr;}@@ -7664,7 +7765,7 @@ static void ggml_compute_forward_mul_f32(const struct ggml_tensor * src0,const struct ggml_tensor * src1,struct ggml_tensor * dst) {- GGML_ASSERT(ggml_can_repeat_rows(src1, src0) && ggml_are_same_shape(src0, dst));+ GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;@@ -7687,7 +7788,6 @@ static void ggml_compute_forward_mul_f32(GGML_ASSERT( nb0 == sizeof(float));GGML_ASSERT(nb00 == sizeof(float));- GGML_ASSERT(ne00 == ne10);if (nb10 == sizeof(float)) {for (int64_t ir = ith; ir < nr; ir += nth) {@@ -7699,20 +7799,21 @@ static void ggml_compute_forward_mul_f32(const int64_t i13 = i03 % ne13;const int64_t i12 = i02 % ne12;const int64_t i11 = i01 % ne11;+ const int64_t nr0 = ne00 / ne10;float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);+ for (int64_t r = 0 ; r < nr0; ++r) {#ifdef GGML_USE_ACCELERATE- UNUSED(ggml_vec_mul_f32);+ UNUSED(ggml_vec_mul_f32);- vDSP_vmul( src0_ptr, 1, src1_ptr, 1, dst_ptr, 1, ne00);+ vDSP_vmul(src0_ptr + r*ne10, 1, src1_ptr, 1, dst_ptr + r*ne10, 1, ne10);#else- ggml_vec_mul_f32(ne00, dst_ptr, src0_ptr, src1_ptr);+ ggml_vec_mul_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);#endif- // }- // }+ }}} else {// src1 is not contiguous@@ -7730,8 +7831,9 @@ static void ggml_compute_forward_mul_f32(float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);- for (int64_t i0 = 0; i0 < ne00; i0++) {- float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i0*nb10);+ for (int64_t i0 = 0; i0 < ne00; ++i0) {+ const int64_t i10 = i0 % ne10;+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);dst_ptr[i0] = src0_ptr[i0] * (*src1_ptr);}@@ -7765,14 +7867,16 @@ static void ggml_compute_forward_div_f32(const struct ggml_tensor * src0,const struct ggml_tensor * src1,struct ggml_tensor * dst) {- assert(params->ith == 0);- assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));+ GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}- const int nr = ggml_nrows(src0);+ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t nr = ggml_nrows(src0);GGML_TENSOR_BINARY_OP_LOCALS@@ -7780,41 +7884,50 @@ static void ggml_compute_forward_div_f32(GGML_ASSERT(nb00 == sizeof(float));if (nb10 == sizeof(float)) {- for (int ir = 0; ir < nr; ++ir) {- // src0, src1 and dst are same shape => same indices- const int i3 = ir/(ne2*ne1);- const int i2 = (ir - i3*ne2*ne1)/ne1;- const int i1 = (ir - i3*ne2*ne1 - i2*ne1);+ for (int64_t ir = ith; ir < nr; ir += nth) {+ // src0 and dst are same shape => same indices+ const int64_t i03 = ir/(ne02*ne01);+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);++ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;+ const int64_t nr0 = ne00 / ne10;+ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);++ for (int64_t r = 0; r < nr0; ++r) {#ifdef GGML_USE_ACCELERATE- UNUSED(ggml_vec_div_f32);+ UNUSED(ggml_vec_div_f32);- vDSP_vdiv(- (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11), 1,- (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), 1,- (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), 1,- ne0);+ vDSP_vdiv(src1_ptr, 1, src0_ptr + r*ne10, 1, dst_ptr + r*ne10, 1, ne10);#else- ggml_vec_div_f32(ne0,- (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),- (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),- (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));+ ggml_vec_div_f32(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);#endif- // }- // }+ }}} else {// src1 is not contiguous- for (int ir = 0; ir < nr; ++ir) {- // src0, src1 and dst are same shape => same indices- const int i3 = ir/(ne2*ne1);- const int i2 = (ir - i3*ne2*ne1)/ne1;- const int i1 = (ir - i3*ne2*ne1 - i2*ne1);+ for (int64_t ir = ith; ir < nr; ir += nth) {+ // src0 and dst are same shape => same indices+ // src1 is broadcastable across src0 and dst in i1, i2, i3+ const int64_t i03 = ir/(ne02*ne01);+ const int64_t i02 = (ir - i03*ne02*ne01)/ne01;+ const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);- float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );- float * src0_ptr = (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);- for (int i0 = 0; i0 < ne0; i0++) {- float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11 + i0*nb10);+ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;++ float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );+ float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);++ for (int64_t i0 = 0; i0 < ne00; ++i0) {+ const int64_t i10 = i0 % ne10;+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);dst_ptr[i0] = src0_ptr[i0] / (*src1_ptr);}@@ -8260,7 +8373,7 @@ static void ggml_compute_forward_repeat_f16(return;}- GGML_TENSOR_UNARY_OP_LOCALS;+ GGML_TENSOR_UNARY_OP_LOCALS// guaranteed to be an integer due to the check in ggml_can_repeatconst int nr0 = (int)(ne0/ne00);@@ -8405,6 +8518,7 @@ static void ggml_compute_forward_concat_f32(GGML_ASSERT(src0->nb[0] == sizeof(float));const int ith = params->ith;+ const int nth = params->nth;GGML_TENSOR_BINARY_OP_LOCALS@@ -8414,7 +8528,7 @@ static void ggml_compute_forward_concat_f32(GGML_ASSERT(nb10 == sizeof(float));for (int i3 = 0; i3 < ne3; i3++) {- for (int i2 = ith; i2 < ne2; i2++) {+ for (int i2 = ith; i2 < ne2; i2 += nth) {if (i2 < ne02) { // src0for (int i1 = 0; i1 < ne1; i1++) {for (int i0 = 0; i0 < ne0; i0++) {@@ -8927,6 +9041,48 @@ static void ggml_compute_forward_silu(}}+// ggml_compute_forward_leaky++static void ggml_compute_forward_leaky_f32(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {+ assert(params->ith == 0);+ assert(ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {+ return;+ }++ const int n = ggml_nrows(src0);+ const int nc = src0->ne[0];++ assert(dst->nb[0] == sizeof(float));+ assert(src0->nb[0] == sizeof(float));++ for (int i = 0; i < n; i++) {+ ggml_vec_leaky_f32(nc,+ (float *) ((char *) dst->data + i*( dst->nb[1])),+ (float *) ((char *) src0->data + i*(src0->nb[1])));+ }+}++static void ggml_compute_forward_leaky(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {+ switch (src0->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_leaky_f32(params, src0, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}+// ggml_compute_forward_silu_backstatic void ggml_compute_forward_silu_back_f32(@@ -9407,9 +9563,14 @@ static bool ggml_compute_forward_mul_mat_use_blas(const int64_t ne0 = dst->ne[0];const int64_t ne1 = dst->ne[1];+ // NOTE: with GGML_OP_MUL_MAT_ID we don't want to go through the BLAS branch because it will dequantize (to_float)+ // all the experts for each batch element and the processing would become incredibly slow// TODO: find the optimal values for these- if (ggml_is_contiguous(src0) &&+ if (dst->op != GGML_OP_MUL_MAT_ID &&+ ggml_is_contiguous(src0) &&ggml_is_contiguous(src1) &&+ //src0->type == GGML_TYPE_F32 &&+ src1->type == GGML_TYPE_F32 &&(ne0 >= 32 && ne1 >= 32 && ne10 >= 32)) {/*printf("BLAS: %d %d %d %d %d\n", ne0, ne1, ne10, ne00, ne01);*/@@ -9420,11 +9581,16 @@ static bool ggml_compute_forward_mul_mat_use_blas(}#endif+// off1 = offset in i11 and i1+// cne1 = ne11 and ne1+// in a normal matrix multiplication, off1 = 0 and cne1 = ne1+// during GGML_TASK_INIT, the full src1 is converted regardless of off1 and cne1static void ggml_compute_forward_mul_mat(const struct ggml_compute_params * params,const struct ggml_tensor * src0,const struct ggml_tensor * src1,- struct ggml_tensor * dst) {+ struct ggml_tensor * dst,+ int64_t off1, int64_t cne1) {int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -9448,7 +9614,7 @@ static void ggml_compute_forward_mul_mat(// we don't support permuted src0 or src1GGML_ASSERT(nb00 == ggml_type_size(type));- GGML_ASSERT(nb10 == sizeof(float));+ GGML_ASSERT(nb10 == ggml_type_size(src1->type));// dst cannot be transposed or permutedGGML_ASSERT(nb0 == sizeof(float));@@ -9492,10 +9658,9 @@ static void ggml_compute_forward_mul_mat(const int64_t i03 = i13/r3;const int64_t i02 = i12/r2;- const void * x = (char *) src0->data + i02*nb02 + i03*nb03;- const float * y = (float *) ((char *) src1->data + i12*nb12 + i13*nb13);-- float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);+ const void * x = (char *) src0->data + i02*nb02 + i03*nb03;+ const float * y = (float *) ((char *) src1->data + off1*nb11 + i12*nb12 + i13*nb13);+ float * d = (float *) ((char *) dst->data + off1*nb1 + i12*nb2 + i13*nb3);if (type != GGML_TYPE_F32) {float * const wdata = params->wdata;@@ -9512,10 +9677,10 @@ static void ggml_compute_forward_mul_mat(}cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,- ne11, ne01, ne10,- 1.0f, y, ne10,- x, ne00,- 0.0f, d, ne01);+ cne1, ne01, ne10,+ 1.0f, y, ne10,+ x, ne00,+ 0.0f, d, ne01);}}@@ -9530,6 +9695,9 @@ static void ggml_compute_forward_mul_mat(char * wdata = params->wdata;const size_t row_size = ne10*ggml_type_size(vec_dot_type)/ggml_blck_size(vec_dot_type);+ assert(params->wsize >= ne11*ne12*ne13*row_size);+ assert(src1->type == GGML_TYPE_F32);+for (int64_t i13 = 0; i13 < ne13; ++i13) {for (int64_t i12 = 0; i12 < ne12; ++i12) {for (int64_t i11 = 0; i11 < ne11; ++i11) {@@ -9551,7 +9719,7 @@ static void ggml_compute_forward_mul_mat(const size_t row_size = ne10*ggml_type_size(vec_dot_type)/ggml_blck_size(vec_dot_type);const int64_t nr0 = ne01; // src0 rows- const int64_t nr1 = ne11*ne12*ne13; // src1 rows+ const int64_t nr1 = cne1*ne12*ne13; // src1 rows//printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1);@@ -9593,9 +9761,9 @@ static void ggml_compute_forward_mul_mat(for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) {for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) {for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ++ir1) {- const int64_t i13 = (ir1/(ne12*ne11));- const int64_t i12 = (ir1 - i13*ne12*ne11)/ne11;- const int64_t i11 = (ir1 - i13*ne12*ne11 - i12*ne11);+ const int64_t i13 = (ir1/(ne12*cne1));+ const int64_t i12 = (ir1 - i13*ne12*cne1)/cne1;+ const int64_t i11 = (ir1 - i13*ne12*cne1 - i12*cne1) + off1;// broadcast src0 into src1const int64_t i03 = i13/r3;@@ -9631,6 +9799,34 @@ static void ggml_compute_forward_mul_mat(}}+// ggml_compute_forward_mul_mat_id++static void ggml_compute_forward_mul_mat_id(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ struct ggml_tensor * dst) {++ if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {+ // during GGML_TASK_INIT the entire src1 is converted to vec_dot_type+ ggml_compute_forward_mul_mat(params, dst->src[2], src1, dst, 0, dst->ne[1]);+ return;+ }++ const struct ggml_tensor * ids = src0;+ const int id = ggml_get_op_params_i32(dst, 0);+ const int n_as = ggml_get_op_params_i32(dst, 1);++ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ const int32_t row_id = *(const int32_t *) ((const char *) ids->data + i01*ids->nb[1] + id*ids->nb[0]);++ GGML_ASSERT(row_id >= 0 && row_id < n_as);++ const struct ggml_tensor * src0_row = dst->src[row_id + 2];+ ggml_compute_forward_mul_mat(params, src0_row, src1, dst, i01, 1);+ }+}+// ggml_compute_forward_out_prodstatic void ggml_compute_forward_out_prod_f32(@@ -9646,10 +9842,12 @@ static void ggml_compute_forward_out_prod_f32(const int ith = params->ith;const int nth = params->nth;+ GGML_ASSERT(ne0 == ne00);+ GGML_ASSERT(ne1 == ne10);+ GGML_ASSERT(ne2 == ne02);GGML_ASSERT(ne02 == ne12);- GGML_ASSERT(ne03 == ne13);- GGML_ASSERT(ne2 == ne12);GGML_ASSERT(ne3 == ne13);+ GGML_ASSERT(ne03 == ne13);// we don't support permuted src0 or src1GGML_ASSERT(nb00 == sizeof(float));@@ -9660,18 +9858,25 @@ static void ggml_compute_forward_out_prod_f32(// GGML_ASSERT(nb1 <= nb2);// GGML_ASSERT(nb2 <= nb3);- GGML_ASSERT(ne0 == ne00);- GGML_ASSERT(ne1 == ne10);- GGML_ASSERT(ne2 == ne02);- GGML_ASSERT(ne3 == ne03);-// nb01 >= nb00 - src0 is not transposed// compute by src0 rows// TODO: #if defined(GGML_USE_CUBLAS) ggml_cuda_out_prod- // TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST)+ // TODO: #if defined(GGML_USE_CLBLAST)++#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)+ bool use_blas = ggml_is_matrix(src0) &&+ ggml_is_matrix(src1) &&+ ggml_is_contiguous(src0) &&+ (ggml_is_contiguous(src1) || ggml_is_transposed(src1));+#endifif (params->type == GGML_TASK_INIT) {+#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) // gemm beta will zero dst+ if (use_blas) {+ return;+ }+#endifggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);return;}@@ -9680,6 +9885,50 @@ static void ggml_compute_forward_out_prod_f32(return;}+#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)+ if (use_blas) {+ if (params->ith != 0) { // All threads other than the first do no work.+ return;+ }+ // Arguments to ggml_compute_forward_out_prod (expressed as major,minor)+ // src0: (k,n)+ // src1: (k,m)+ // dst: (m,n)+ //+ // Arguments to sgemm (see https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/sgemm.f)+ // Also expressed as (major,minor)+ // a: (m,k): so src1 transposed+ // b: (k,n): so src0+ // c: (m,n)+ //+ // However, if ggml_is_transposed(src1) is true, then+ // src1->data already contains a transposed version, so sgemm mustn't+ // transpose it further.++ int n = src0->ne[0];+ int k = src0->ne[1];+ int m = src1->ne[0];++ int transposeA, lda;++ if (!ggml_is_transposed(src1)) {+ transposeA = CblasTrans;+ lda = m;+ } else {+ transposeA = CblasNoTrans;+ lda = k;+ }++ float * a = (float *) ((char *) src1->data);+ float * b = (float *) ((char *) src0->data);+ float * c = (float *) ((char *) dst->data);++ cblas_sgemm(CblasRowMajor, transposeA, CblasNoTrans, m, n, k, 1.0, a, lda, b, n, 0.0, c, n);++ return;+ }+#endif+// dst[:,:,:,:] = 0// for i2,i3:// for i1:@@ -10151,21 +10400,30 @@ static void ggml_compute_forward_get_rows_q(return;}- const int nc = src0->ne[0];- const int nr = ggml_nelements(src1);+ GGML_TENSOR_BINARY_OP_LOCALS++ const int64_t nc = ne00;+ const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr);+const enum ggml_type type = src0->type;ggml_to_float_t const dequantize_row_q = type_traits[type].to_float;- assert( dst->ne[0] == nc);- assert( dst->ne[1] == nr);- assert(src0->nb[0] == ggml_type_size(type));+ assert(ne0 == nc);+ assert(ne02 == ne11);+ assert(nb00 == ggml_type_size(type));+ assert(ggml_nrows(dst) == nr);- for (int i = 0; i < nr; ++i) {- const int r = ((int32_t *) src1->data)[i];+ // TODO: multi-thread+ for (int64_t i12 = 0; i12 < ne12; ++i12) {+ for (int64_t i11 = 0; i11 < ne11; ++i11) {+ for (int64_t i10 = 0; i10 < ne10; ++i10) {+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);- dequantize_row_q(- (const void *) ((char *) src0->data + r*src0->nb[1]),- (float *) ((char *) dst->data + i*dst->nb[1]), nc);+ dequantize_row_q(+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);+ }+ }}}@@ -10180,19 +10438,26 @@ static void ggml_compute_forward_get_rows_f16(return;}- const int nc = src0->ne[0];- const int nr = ggml_nelements(src1);+ GGML_TENSOR_BINARY_OP_LOCALS- assert( dst->ne[0] == nc);- assert( dst->ne[1] == nr);- assert(src0->nb[0] == sizeof(ggml_fp16_t));+ const int64_t nc = ne00;+ const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr);- for (int i = 0; i < nr; ++i) {- const int r = ((int32_t *) src1->data)[i];+ assert(ne0 == nc);+ assert(ne02 == ne11);+ assert(nb00 == sizeof(ggml_fp16_t));+ assert(ggml_nrows(dst) == nr);- for (int j = 0; j < nc; ++j) {- ggml_fp16_t v = ((ggml_fp16_t *) ((char *) src0->data + r*src0->nb[1]))[j];- ((float *) ((char *) dst->data + i*dst->nb[1]))[j] = GGML_FP16_TO_FP32(v);+ // TODO: multi-thread+ for (int64_t i12 = 0; i12 < ne12; ++i12) {+ for (int64_t i11 = 0; i11 < ne11; ++i11) {+ for (int64_t i10 = 0; i10 < ne10; ++i10) {+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);++ ggml_fp16_to_fp32_row(+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);+ }}}}@@ -10208,19 +10473,27 @@ static void ggml_compute_forward_get_rows_f32(return;}- const int nc = src0->ne[0];- const int nr = ggml_nelements(src1);+ GGML_TENSOR_BINARY_OP_LOCALS- assert( dst->ne[0] == nc);- assert( dst->ne[1] == nr);- assert(src0->nb[0] == sizeof(float));+ const int64_t nc = ne00;+ const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr);- for (int i = 0; i < nr; ++i) {- const int r = ((int32_t *) src1->data)[i];+ assert(ne0 == nc);+ assert(ne02 == ne11);+ assert(nb00 == sizeof(float));+ assert(ggml_nrows(dst) == nr);- ggml_vec_cpy_f32(nc,- (float *) ((char *) dst->data + i*dst->nb[1]),- (float *) ((char *) src0->data + r*src0->nb[1]));+ // TODO: multi-thread+ for (int64_t i12 = 0; i12 < ne12; ++i12) {+ for (int64_t i11 = 0; i11 < ne11; ++i11) {+ for (int64_t i10 = 0; i10 < ne10; ++i10) {+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);++ ggml_vec_cpy_f32(nc,+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3),+ (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03));+ }+ }}}@@ -10533,20 +10806,25 @@ static void ggml_compute_forward_diag_mask_zero(static void ggml_compute_forward_soft_max_f32(const struct ggml_compute_params * params,const struct ggml_tensor * src0,- struct ggml_tensor * dst) {- GGML_ASSERT(ggml_is_contiguous(src0));- GGML_ASSERT(ggml_is_contiguous(dst));- GGML_ASSERT(ggml_are_same_shape(src0, dst));+ const struct ggml_tensor * src1,+ struct ggml_tensor * dst) {+ assert(ggml_is_contiguous(dst));+ assert(ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}+ float scale = 1.0f;+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));+// TODO: handle transposed/permuted matricesconst int ith = params->ith;const int nth = params->nth;+ const int64_t ne11 = src1 ? src1->ne[1] : 1;+const int nc = src0->ne[0];const int nr = ggml_nrows(src0);@@ -10557,29 +10835,40 @@ static void ggml_compute_forward_soft_max_f32(const int ir0 = dr*ith;const int ir1 = MIN(ir0 + dr, nr);+ float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;+for (int i1 = ir0; i1 < ir1; i1++) {- float *sp = (float *)((char *) src0->data + i1*src0->nb[1]);- float *dp = (float *)((char *) dst->data + i1*dst->nb[1]);+ float * sp = (float *)((char *) src0->data + i1*src0->nb[1]);+ float * dp = (float *)((char *) dst->data + i1*dst->nb[1]);++ // broadcast the mask across rows+ float * mp = src1 ? (float *)((char *) src1->data + (i1%ne11)*src1->nb[1]) : NULL;++ ggml_vec_cpy_f32 (nc, wp, sp);+ ggml_vec_scale_f32(nc, wp, scale);+ if (mp) {+ ggml_vec_acc_f32(nc, wp, mp);+ }#ifndef NDEBUGfor (int i = 0; i < nc; ++i) {//printf("p[%d] = %f\n", i, p[i]);- assert(!isnan(sp[i]));+ assert(!isnan(wp[i]));}#endiffloat max = -INFINITY;- ggml_vec_max_f32(nc, &max, sp);+ ggml_vec_max_f32(nc, &max, wp);ggml_float sum = 0.0;uint16_t scvt;for (int i = 0; i < nc; i++) {- if (sp[i] == -INFINITY) {+ if (wp[i] == -INFINITY) {dp[i] = 0.0f;} else {- // const float val = (sp[i] == -INFINITY) ? 0.0 : exp(sp[i] - max);- ggml_fp16_t s = GGML_FP32_TO_FP16(sp[i] - max);+ // const float val = (wp[i] == -INFINITY) ? 0.0 : exp(wp[i] - max);+ ggml_fp16_t s = GGML_FP32_TO_FP16(wp[i] - max);memcpy(&scvt, &s, sizeof(scvt));const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt]);sum += (ggml_float)val;@@ -10604,11 +10893,12 @@ static void ggml_compute_forward_soft_max_f32(static void ggml_compute_forward_soft_max(const struct ggml_compute_params * params,const struct ggml_tensor * src0,- struct ggml_tensor * dst) {+ const struct ggml_tensor * src1,+ struct ggml_tensor * dst) {switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_soft_max_f32(params, src0, dst);+ ggml_compute_forward_soft_max_f32(params, src0, src1, dst);} break;default:{@@ -11346,9 +11636,9 @@ static void ggml_compute_forward_rope_back(}}-// ggml_compute_forward_conv_1d+// ggml_compute_forward_conv_transpose_1d-static void ggml_compute_forward_conv_1d_f16_f32(+static void ggml_compute_forward_conv_transpose_1d_f16_f32(const struct ggml_compute_params * params,const struct ggml_tensor * src0,const struct ggml_tensor * src1,@@ -11365,14 +11655,7 @@ static void ggml_compute_forward_conv_1d_f16_f32(const int ith = params->ith;const int nth = params->nth;- const int nk = ne00;-- // size of the convolution row - the kernel size unrolled across all input channels- const int ew0 = nk*ne01;-- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];- const int32_t p0 = ((const int32_t*)(dst->op_params))[1];- const int32_t d0 = ((const int32_t*)(dst->op_params))[2];+ const int nk = ne00*ne01*ne02;GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == sizeof(float));@@ -11380,23 +11663,37 @@ static void ggml_compute_forward_conv_1d_f16_f32(if (params->type == GGML_TASK_INIT) {memset(params->wdata, 0, params->wsize);- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;+ // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)+ {+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;- for (int64_t i11 = 0; i11 < ne11; i11++) {- const float * const src = (float *)((char *) src1->data + i11*nb11);- ggml_fp16_t * dst_data = wdata;+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);+ ggml_fp16_t * dst_data = wdata + i01*ne00*ne02;+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ dst_data[i00*ne02 + i02] = src[i00];+ }+ }+ }+ }- for (int64_t i0 = 0; i0 < ne0; i0++) {- for (int64_t ik = 0; ik < nk; ik++) {- const int idx0 = i0*s0 + ik*d0 - p0;+ // permute source data (src1) from (L x Cin) to (Cin x L)+ {+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;+ ggml_fp16_t * dst_data = wdata;- if(!(idx0 < 0 || idx0 >= ne10)) {- dst_data[i0*ew0 + i11*nk + ik] = GGML_FP32_TO_FP16(src[idx0]);- }+ for (int64_t i11 = 0; i11 < ne11; i11++) {+ const float * const src = (float *)((char *) src1->data + i11*nb11);+ for (int64_t i10 = 0; i10 < ne10; i10++) {+ dst_data[i10*ne11 + i11] = GGML_FP32_TO_FP16(src[i10]);}}}+ // need to zero dst since we are accumulating into it+ memset(dst->data, 0, ggml_nbytes(dst));+return;}@@ -11404,8 +11701,10 @@ static void ggml_compute_forward_conv_1d_f16_f32(return;}+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];+// total rows in dst- const int nr = ne2;+ const int nr = ne1;// rows per threadconst int dr = (nr + nth - 1)/nth;@@ -11414,22 +11713,26 @@ static void ggml_compute_forward_conv_1d_f16_f32(const int ir0 = dr*ith;const int ir1 = MIN(ir0 + dr, nr);- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;-- for (int i2 = 0; i2 < ne2; i2++) {- for (int i1 = ir0; i1 < ir1; i1++) {- float * dst_data = (float *)((char *) dst->data + i2*nb2 + i1*nb1);+ ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;+ ggml_fp16_t * const wdata_src = wdata + nk;- for (int i0 = 0; i0 < ne0; i0++) {- ggml_vec_dot_f16(ew0, dst_data + i0,- (ggml_fp16_t *) ((char *) src0->data + i1*nb02),- (ggml_fp16_t *) wdata + i2*nb2 + i0*ew0);+ for (int i1 = ir0; i1 < ir1; i1++) {+ float * dst_data = (float *)((char *) dst->data + i1*nb1);+ ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00;+ for (int i10 = 0; i10 < ne10; i10++) {+ const int i1n = i10*ne11;+ for (int i00 = 0; i00 < ne00; i00++) {+ float v = 0;+ ggml_vec_dot_f16(ne02, &v,+ (ggml_fp16_t *) wdata_src + i1n,+ (ggml_fp16_t *) wdata_kernel + i00*ne02);+ dst_data[i10*s0 + i00] += v;}}}}-static void ggml_compute_forward_conv_1d_f32(+static void ggml_compute_forward_conv_transpose_1d_f32(const struct ggml_compute_params * params,const struct ggml_tensor * src0,const struct ggml_tensor * src1,@@ -11446,13 +11749,7 @@ static void ggml_compute_forward_conv_1d_f32(const int ith = params->ith;const int nth = params->nth;- const int nk = ne00;-- const int ew0 = nk*ne01;-- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];- const int32_t p0 = ((const int32_t*)(dst->op_params))[1];- const int32_t d0 = ((const int32_t*)(dst->op_params))[2];+ const int nk = ne00*ne01*ne02;GGML_ASSERT(nb00 == sizeof(float));GGML_ASSERT(nb10 == sizeof(float));@@ -11460,23 +11757,37 @@ static void ggml_compute_forward_conv_1d_f32(if (params->type == GGML_TASK_INIT) {memset(params->wdata, 0, params->wsize);- float * const wdata = (float *) params->wdata + 0;+ // prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)+ {+ float * const wdata = (float *) params->wdata + 0;- for (int64_t i11 = 0; i11 < ne11; i11++) {- const float * const src = (float *)((char *) src1->data + i11*nb11);- float * dst_data = wdata;+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);+ float * dst_data = wdata + i01*ne00*ne02;+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ dst_data[i00*ne02 + i02] = src[i00];+ }+ }+ }+ }- for (int64_t i0 = 0; i0 < ne0; i0++) {- for (int64_t ik = 0; ik < nk; ik++) {- const int idx0 = i0*s0 + ik*d0 - p0;+ // prepare source data (src1)+ {+ float * const wdata = (float *) params->wdata + nk;+ float * dst_data = wdata;- if(!(idx0 < 0 || idx0 >= ne10)) {- dst_data[i0*ew0 + i11*nk + ik] = src[idx0];- }+ for (int64_t i11 = 0; i11 < ne11; i11++) {+ const float * const src = (float *)((char *) src1->data + i11*nb11);+ for (int64_t i10 = 0; i10 < ne10; i10++) {+ dst_data[i10*ne11 + i11] = src[i10];}}}+ // need to zero dst since we are accumulating into it+ memset(dst->data, 0, ggml_nbytes(dst));+return;}@@ -11484,8 +11795,10 @@ static void ggml_compute_forward_conv_1d_f32(return;}+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];+// total rows in dst- const int nr = ne02;+ const int nr = ne1;// rows per threadconst int dr = (nr + nth - 1)/nth;@@ -11494,94 +11807,50 @@ static void ggml_compute_forward_conv_1d_f32(const int ir0 = dr*ith;const int ir1 = MIN(ir0 + dr, nr);- float * const wdata = (float *) params->wdata + 0;-- for (int i2 = 0; i2 < ne2; i2++) {- for (int i1 = ir0; i1 < ir1; i1++) {- float * dst_data = (float *)((char *) dst->data + i2*nb2 + i1*nb1);+ float * const wdata = (float *) params->wdata + 0;+ float * const wdata_src = wdata + nk;- for (int i0 = 0; i0 < ne0; i0++) {- ggml_vec_dot_f32(ew0, dst_data + i0,- (float *) ((char *) src0->data + i1*nb02),- (float *) wdata + i2*nb2 + i0*ew0);+ for (int i1 = ir0; i1 < ir1; i1++) {+ float * dst_data = (float *)((char *) dst->data + i1*nb1);+ float * wdata_kernel = wdata + i1*ne02*ne00;+ for (int i10 = 0; i10 < ne10; i10++) {+ const int i1n = i10*ne11;+ for (int i00 = 0; i00 < ne00; i00++) {+ float v = 0;+ ggml_vec_dot_f32(ne02, &v,+ wdata_src + i1n,+ wdata_kernel + i00*ne02);+ dst_data[i10*s0 + i00] += v;}}}}-// TODO: reuse ggml_mul_mat or implement ggml_im2col and remove stage_0 and stage_1-static void gemm_f16_out_f32(int64_t m, int64_t n, int64_t k,- ggml_fp16_t * A,- ggml_fp16_t * B,- float * C,- const int ith, const int nth) {- // does not seem to make a difference- int64_t m0, m1, n0, n1;- // patches per thread- if (m > n) {- n0 = 0;- n1 = n;-- // total patches in dst- const int np = m;-- // patches per thread- const int dp = (np + nth - 1)/nth;-- // patch range for this thread- m0 = dp*ith;- m1 = MIN(m0 + dp, np);- } else {- m0 = 0;- m1 = m;-- // total patches in dst- const int np = n;-- // patches per thread- const int dp = (np + nth - 1)/nth;-- // patch range for this thread- n0 = dp*ith;- n1 = MIN(n0 + dp, np);- }-- // block-tiling attempt- int64_t blck_n = 16;- int64_t blck_m = 16;-- // int64_t CACHE_SIZE = 2 * 1024 * 1024; // 2MB- // int64_t blck_size = CACHE_SIZE / (sizeof(float) + 2 * sizeof(ggml_fp16_t) * K);- // if (blck_size > 0) {- // blck_0 = 4;- // blck_1 = blck_size / blck_0;- // if (blck_1 < 0) {- // blck_1 = 1;- // }- // // blck_0 = (int64_t)sqrt(blck_size);- // // blck_1 = blck_0;- // }- // // printf("%zd %zd %zd %zd\n", blck_size, K, blck_0, blck_1);-- for (int j = n0; j < n1; j+=blck_n) {- for (int i = m0; i < m1; i+=blck_m) {- // printf("i j k => %d %d %d\n", i, j, K);- for (int ii = i; ii < i + blck_m && ii < m1; ii++) {- for (int jj = j; jj < j + blck_n && jj < n1; jj++) {- ggml_vec_dot_f16(k,- C + ii*n + jj,- A + ii * k,- B + jj * k);- }- }- }+static void ggml_compute_forward_conv_transpose_1d(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ struct ggml_tensor * dst) {+ switch (src0->type) {+ case GGML_TYPE_F16:+ {+ ggml_compute_forward_conv_transpose_1d_f16_f32(params, src0, src1, dst);+ } break;+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_conv_transpose_1d_f32(params, src0, src1, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;}}-// src0: kernel [OC, IC, K]-// src1: signal [N, IC, IL]-// dst: result [N, OL, IC*K]-static void ggml_compute_forward_conv_1d_stage_0_f32(+// src0: kernel [OC, IC, KH, KW]+// src1: image [N, IC, IH, IW]+// dst: result [N, OH, OW, IC*KH*KW]+static void ggml_compute_forward_im2col_f16(const struct ggml_compute_params * params,const struct ggml_tensor * src0,const struct ggml_tensor * src1,@@ -11595,26 +11864,35 @@ static void ggml_compute_forward_conv_1d_stage_0_f32(GGML_TENSOR_BINARY_OP_LOCALS;- const int64_t N = ne12;- const int64_t IC = ne11;- const int64_t IL = ne10;-- const int64_t K = ne00;-- const int64_t OL = ne1;+ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;const int ith = params->ith;const int nth = params->nth;- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];- const int32_t p0 = ((const int32_t*)(dst->op_params))[1];- const int32_t d0 = ((const int32_t*)(dst->op_params))[2];+ const int64_t N = is_2D ? ne13 : ne12;+ const int64_t IC = is_2D ? ne12 : ne11;+ const int64_t IH = is_2D ? ne11 : 1;+ const int64_t IW = ne10;++ const int64_t KH = is_2D ? ne01 : 1;+ const int64_t KW = ne00;++ const int64_t OH = is_2D ? ne2 : 1;+ const int64_t OW = ne1;++ int ofs0 = is_2D ? nb13 : nb12;+ int ofs1 = is_2D ? nb12 : nb11;GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == sizeof(float));if (params->type == GGML_TASK_INIT) {- memset(dst->data, 0, ggml_nbytes(dst));return;}@@ -11622,23 +11900,30 @@ static void ggml_compute_forward_conv_1d_stage_0_f32(return;}- // im2col: [N, IC, IL] => [N, OL, IC*K]+ // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]{ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;for (int64_t in = 0; in < N; in++) {- for (int64_t iol = 0; iol < OL; iol++) {- for (int64_t iic = ith; iic < IC; iic+=nth) {+ for (int64_t ioh = 0; ioh < OH; ioh++) { // 1+ for (int64_t iow = 0; iow < OW; iow++) {+ for (int64_t iic = ith; iic < IC; iic += nth) {- // micro kernel- ggml_fp16_t * dst_data = wdata + (in*OL + iol)*(IC*K); // [IC, K]- const float * const src_data = (float *)((char *) src1->data + in*nb12 + iic*nb11); // [IL]+ // micro kernel+ ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]+ const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]- for (int64_t ik = 0; ik < K; ik++) {- const int64_t iil = iol*s0 + ik*d0 - p0;+ for (int64_t ikh = 0; ikh < KH; ikh++) { // 1+ for (int64_t ikw = 0; ikw < KW; ikw++) {+ const int64_t iiw = iow*s0 + ikw*d0 - p0;+ const int64_t iih = ioh*s1 + ikh*d1 - p1;- if (!(iil < 0 || iil >= IL)) {- dst_data[iic*K + ik] = GGML_FP32_TO_FP16(src_data[iil]);+ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;+ } else {+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);+ }+ }}}}@@ -11647,108 +11932,20 @@ static void ggml_compute_forward_conv_1d_stage_0_f32(}}-// gemm: [N, OC, OL] = [OC, IC * K] x [N*OL, IC * K]-// src0: [OC, IC, K]-// src1: [N, OL, IC * K]-// result: [N, OC, OL]-static void ggml_compute_forward_conv_1d_stage_1_f16(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F16);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- int64_t t0 = ggml_perf_time_us();- UNUSED(t0);-- if (params->type == GGML_TASK_INIT) {- return;- }-- if (params->type == GGML_TASK_FINALIZE) {- return;- }-- GGML_TENSOR_BINARY_OP_LOCALS;-- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb10 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb0 == sizeof(float));-- const int N = ne12;- const int OL = ne11;-- const int OC = ne02;- const int IC = ne01;- const int K = ne00;-- const int ith = params->ith;- const int nth = params->nth;-- int64_t m = OC;- int64_t n = OL;- int64_t k = IC * K;-- // [N, OC, OL] = [OC, IC * K] x [N*OL, IC * K]- for (int i = 0; i < N; i++) {- ggml_fp16_t * A = (ggml_fp16_t *)src0->data; // [m, k]- ggml_fp16_t * B = (ggml_fp16_t *)src1->data + i * m * k; // [n, k]- float * C = (float *)dst->data + i * m * n; // [m, n]-- gemm_f16_out_f32(m, n, k, A, B, C, ith, nth);- }-}--static void ggml_compute_forward_conv_1d(+static void ggml_compute_forward_im2col(const struct ggml_compute_params * params,const struct ggml_tensor * src0,const struct ggml_tensor * src1,struct ggml_tensor * dst) {- switch(src0->type) {+ switch (src0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_conv_1d_f16_f32(params, src0, src1, dst);+ ggml_compute_forward_im2col_f16(params, src0, src1, dst);} break;case GGML_TYPE_F32:- {- ggml_compute_forward_conv_1d_f32(params, src0, src1, dst);- } break;- default:- {- GGML_ASSERT(false);- } break;- }-}--static void ggml_compute_forward_conv_1d_stage_0(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- switch(src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_conv_1d_stage_0_f32(params, src0, src1, dst);- } break;- default:{GGML_ASSERT(false);} break;- }-}--static void ggml_compute_forward_conv_1d_stage_1(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- switch(src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_conv_1d_stage_1_f16(params, src0, src1, dst);- } break;default:{GGML_ASSERT(false);@@ -11756,9 +11953,9 @@ static void ggml_compute_forward_conv_1d_stage_1(}}-// ggml_compute_forward_conv_transpose_1d+// ggml_compute_forward_conv_transpose_2d-static void ggml_compute_forward_conv_transpose_1d_f16_f32(+static void ggml_compute_forward_conv_transpose_2d(const struct ggml_compute_params * params,const struct ggml_tensor * src0,const struct ggml_tensor * src1,@@ -11775,7 +11972,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(const int ith = params->ith;const int nth = params->nth;- const int nk = ne00*ne01*ne02;+ const int nk = ne00*ne01*ne02*ne03;GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == sizeof(float));@@ -11783,568 +11980,36 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(if (params->type == GGML_TASK_INIT) {memset(params->wdata, 0, params->wsize);- // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)+ // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout){ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;- for (int64_t i02 = 0; i02 < ne02; i02++) {- for (int64_t i01 = 0; i01 < ne01; i01++) {- const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);- ggml_fp16_t * dst_data = wdata + i01*ne00*ne02;- for (int64_t i00 = 0; i00 < ne00; i00++) {- dst_data[i00*ne02 + i02] = src[i00];+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i03*nb03 + i02*nb02);+ ggml_fp16_t * dst_data = wdata + i02*ne01*ne00*ne03;+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ dst_data[i01*ne00*ne03 + i00*ne03 + i03] = src[i01 * ne00 + i00];+ }}}}}- // permute source data (src1) from (L x Cin) to (Cin x L)+ // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh){ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;- ggml_fp16_t * dst_data = wdata;-- for (int64_t i11 = 0; i11 < ne11; i11++) {- const float * const src = (float *)((char *) src1->data + i11*nb11);- for (int64_t i10 = 0; i10 < ne10; i10++) {- dst_data[i10*ne11 + i11] = GGML_FP32_TO_FP16(src[i10]);- }- }- }-- // need to zero dst since we are accumulating into it- memset(dst->data, 0, ggml_nbytes(dst));-- return;- }-- if (params->type == GGML_TASK_FINALIZE) {- return;- }-- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];-- // total rows in dst- const int nr = ne1;-- // rows per thread- const int dr = (nr + nth - 1)/nth;-- // row range for this thread- const int ir0 = dr*ith;- const int ir1 = MIN(ir0 + dr, nr);-- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;- ggml_fp16_t * const wdata_src = wdata + nk;-- for (int i1 = ir0; i1 < ir1; i1++) {- float * dst_data = (float *)((char *) dst->data + i1*nb1);- ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00;- for (int i10 = 0; i10 < ne10; i10++) {- const int i1n = i10*ne11;- for (int i00 = 0; i00 < ne00; i00++) {- float v = 0;- ggml_vec_dot_f16(ne02, &v,- (ggml_fp16_t *) wdata_src + i1n,- (ggml_fp16_t *) wdata_kernel + i00*ne02);- dst_data[i10*s0 + i00] += v;- }- }- }-}--static void ggml_compute_forward_conv_transpose_1d_f32(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- int64_t t0 = ggml_perf_time_us();- UNUSED(t0);-- GGML_TENSOR_BINARY_OP_LOCALS-- const int ith = params->ith;- const int nth = params->nth;-- const int nk = ne00*ne01*ne02;-- GGML_ASSERT(nb00 == sizeof(float));- GGML_ASSERT(nb10 == sizeof(float));-- if (params->type == GGML_TASK_INIT) {- memset(params->wdata, 0, params->wsize);-- // prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)- {- float * const wdata = (float *) params->wdata + 0;-- for (int64_t i02 = 0; i02 < ne02; i02++) {- for (int64_t i01 = 0; i01 < ne01; i01++) {- const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);- float * dst_data = wdata + i01*ne00*ne02;- for (int64_t i00 = 0; i00 < ne00; i00++) {- dst_data[i00*ne02 + i02] = src[i00];- }- }- }- }-- // prepare source data (src1)- {- float * const wdata = (float *) params->wdata + nk;- float * dst_data = wdata;-- for (int64_t i11 = 0; i11 < ne11; i11++) {- const float * const src = (float *)((char *) src1->data + i11*nb11);- for (int64_t i10 = 0; i10 < ne10; i10++) {- dst_data[i10*ne11 + i11] = src[i10];- }- }- }-- // need to zero dst since we are accumulating into it- memset(dst->data, 0, ggml_nbytes(dst));-- return;- }-- if (params->type == GGML_TASK_FINALIZE) {- return;- }-- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];-- // total rows in dst- const int nr = ne1;-- // rows per thread- const int dr = (nr + nth - 1)/nth;-- // row range for this thread- const int ir0 = dr*ith;- const int ir1 = MIN(ir0 + dr, nr);-- float * const wdata = (float *) params->wdata + 0;- float * const wdata_src = wdata + nk;-- for (int i1 = ir0; i1 < ir1; i1++) {- float * dst_data = (float *)((char *) dst->data + i1*nb1);- float * wdata_kernel = wdata + i1*ne02*ne00;- for (int i10 = 0; i10 < ne10; i10++) {- const int i1n = i10*ne11;- for (int i00 = 0; i00 < ne00; i00++) {- float v = 0;- ggml_vec_dot_f32(ne02, &v,- wdata_src + i1n,- wdata_kernel + i00*ne02);- dst_data[i10*s0 + i00] += v;- }- }- }-}--static void ggml_compute_forward_conv_transpose_1d(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- switch (src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_conv_transpose_1d_f16_f32(params, src0, src1, dst);- } break;- case GGML_TYPE_F32:- {- ggml_compute_forward_conv_transpose_1d_f32(params, src0, src1, dst);- } break;- default:- {- GGML_ASSERT(false);- } break;- }-}--// ggml_compute_forward_conv_2d--// src0: kernel [OC, IC, KH, KW]-// src1: image [N, IC, IH, IW]-// dst: result [N, OH, OW, IC*KH*KW]-static void ggml_compute_forward_conv_2d_stage_0_f32(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F16);-- int64_t t0 = ggml_perf_time_us();- UNUSED(t0);-- GGML_TENSOR_BINARY_OP_LOCALS;-- const int64_t N = ne13;- const int64_t IC = ne12;- const int64_t IH = ne11;- const int64_t IW = ne10;-- // const int64_t OC = ne03;- // const int64_t IC = ne02;- const int64_t KH = ne01;- const int64_t KW = ne00;-- const int64_t OH = ne2;- const int64_t OW = ne1;-- const int ith = params->ith;- const int nth = params->nth;-- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];- const int32_t s1 = ((const int32_t*)(dst->op_params))[1];- const int32_t p0 = ((const int32_t*)(dst->op_params))[2];- const int32_t p1 = ((const int32_t*)(dst->op_params))[3];- const int32_t d0 = ((const int32_t*)(dst->op_params))[4];- const int32_t d1 = ((const int32_t*)(dst->op_params))[5];-- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb10 == sizeof(float));-- if (params->type == GGML_TASK_INIT) {- memset(dst->data, 0, ggml_nbytes(dst));- return;- }-- if (params->type == GGML_TASK_FINALIZE) {- return;- }-- // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]- {- ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;-- for (int64_t in = 0; in < N; in++) {- for (int64_t ioh = 0; ioh < OH; ioh++) {- for (int64_t iow = 0; iow < OW; iow++) {- for (int64_t iic = ith; iic < IC; iic+=nth) {-- // micro kernel- ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]- const float * const src_data = (float *)((char *) src1->data + in*nb13 + iic*nb12); // [IH, IW]-- for (int64_t ikh = 0; ikh < KH; ikh++) {- for (int64_t ikw = 0; ikw < KW; ikw++) {- const int64_t iiw = iow*s0 + ikw*d0 - p0;- const int64_t iih = ioh*s1 + ikh*d1 - p1;-- if (!(iih < 0 || iih >= IH || iiw < 0 || iiw >= IW)) {- dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);- }- }- }- }- }- }- }- }-}--// gemm: [N, OC, OH, OW] = [OC, IC * KH * KW] x [N*OH*OW, IC * KH * KW]-// src0: [OC, IC, KH, KW]-// src1: [N, OH, OW, IC * KH * KW]-// result: [N, OC, OH, OW]-static void ggml_compute_forward_conv_2d_stage_1_f16(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F16);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- int64_t t0 = ggml_perf_time_us();- UNUSED(t0);-- if (params->type == GGML_TASK_INIT) {- return;- }-- if (params->type == GGML_TASK_FINALIZE) {- return;- }-- GGML_TENSOR_BINARY_OP_LOCALS;-- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb10 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb0 == sizeof(float));-- const int N = ne13;- const int OH = ne12;- const int OW = ne11;-- const int OC = ne03;- const int IC = ne02;- const int KH = ne01;- const int KW = ne00;-- const int ith = params->ith;- const int nth = params->nth;-- int64_t m = OC;- int64_t n = OH * OW;- int64_t k = IC * KH * KW;-- // [N, OC, OH, OW] = [OC, IC * KH * KW] x [N*OH*OW, IC * KH * KW]- for (int i = 0; i < N; i++) {- ggml_fp16_t * A = (ggml_fp16_t *)src0->data; // [m, k]- ggml_fp16_t * B = (ggml_fp16_t *)src1->data + i * m * k; // [n, k]- float * C = (float *)dst->data + i * m * n; // [m, n]-- gemm_f16_out_f32(m, n, k, A, B, C, ith, nth);- }-}--static void ggml_compute_forward_conv_2d_f16_f32(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- int64_t t0 = ggml_perf_time_us();- UNUSED(t0);-- GGML_TENSOR_BINARY_OP_LOCALS-- // src1: image [N, IC, IH, IW]- // src0: kernel [OC, IC, KH, KW]- // dst: result [N, OC, OH, OW]- // ne12: IC- // ne0: OW- // ne1: OH- // nk0: KW- // nk1: KH- // ne13: N-- const int N = ne13;- const int IC = ne12;- const int IH = ne11;- const int IW = ne10;-- const int OC = ne03;- // const int IC = ne02;- const int KH = ne01;- const int KW = ne00;-- const int OH = ne1;- const int OW = ne0;-- const int ith = params->ith;- const int nth = params->nth;-- // const int nk0 = ne00;- // const int nk1 = ne01;-- // size of the convolution row - the kernel size unrolled across all channels- // const int ew0 = nk0*nk1*ne02;- // ew0: IC*KH*KW-- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];- const int32_t s1 = ((const int32_t*)(dst->op_params))[1];- const int32_t p0 = ((const int32_t*)(dst->op_params))[2];- const int32_t p1 = ((const int32_t*)(dst->op_params))[3];- const int32_t d0 = ((const int32_t*)(dst->op_params))[4];- const int32_t d1 = ((const int32_t*)(dst->op_params))[5];-- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb10 == sizeof(float));-- if (params->type == GGML_TASK_INIT) {- memset(params->wdata, 0, params->wsize);-- // prepare source data (src1)- // im2col: [N, IC, IH, IW] => [N*OH*OW, IC*KH*KW]-- {- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;-- for (int in = 0; in < N; in++) {- for (int iic = 0; iic < IC; iic++) {- for (int ioh = 0; ioh < OH; ioh++) {- for (int iow = 0; iow < OW; iow++) {-- // micro kernel- ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]- const float * const src_data = (float *)((char *) src1->data + in*nb13 + iic*nb12); // [IH, IW]-- for (int ikh = 0; ikh < KH; ikh++) {- for (int ikw = 0; ikw < KW; ikw++) {- const int iiw = iow*s0 + ikw*d0 - p0;- const int iih = ioh*s1 + ikh*d1 - p1;-- if (!(iih < 0 || iih >= IH || iiw < 0 || iiw >= IW)) {- dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);- }- }- }- }- }- }- }- }-- return;- }-- if (params->type == GGML_TASK_FINALIZE) {- return;- }-- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;- // wdata: [N*OH*OW, IC*KH*KW]- // dst: result [N, OC, OH, OW]- // src0: kernel [OC, IC, KH, KW]-- int64_t m = OC;- int64_t n = OH * OW;- int64_t k = IC * KH * KW;-- // [N, OC, OH, OW] = [OC, IC * KH * KW] x [N*OH*OW, IC * KH * KW]- for (int i = 0; i < N; i++) {- ggml_fp16_t * A = (ggml_fp16_t *)src0->data; // [m, k]- ggml_fp16_t * B = (ggml_fp16_t *)wdata + i * m * k; // [n, k]- float * C = (float *)dst->data + i * m * n; // [m * k]-- gemm_f16_out_f32(m, n, k, A, B, C, ith, nth);- }-}--static void ggml_compute_forward_conv_2d(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- switch (src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_conv_2d_f16_f32(params, src0, src1, dst);- } break;- case GGML_TYPE_F32:- {- //ggml_compute_forward_conv_2d_f32(params, src0, src1, dst);- GGML_ASSERT(false);- } break;- default:- {- GGML_ASSERT(false);- } break;- }-}--static void ggml_compute_forward_conv_2d_stage_0(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- switch (src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_conv_2d_stage_0_f32(params, src0, src1, dst);- } break;- case GGML_TYPE_F32:- {- GGML_ASSERT(false);- } break;- default:- {- GGML_ASSERT(false);- } break;- }-}--static void ggml_compute_forward_conv_2d_stage_1(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- switch (src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_conv_2d_stage_1_f16(params, src0, src1, dst);- } break;- case GGML_TYPE_F32:- {- GGML_ASSERT(false);- } break;- default:- {- GGML_ASSERT(false);- } break;- }-}--// ggml_compute_forward_conv_transpose_2d--static void ggml_compute_forward_conv_transpose_2d(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- int64_t t0 = ggml_perf_time_us();- UNUSED(t0);-- GGML_TENSOR_BINARY_OP_LOCALS-- const int ith = params->ith;- const int nth = params->nth;-- const int nk = ne00*ne01*ne02*ne03;-- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb10 == sizeof(float));-- if (params->type == GGML_TASK_INIT) {- memset(params->wdata, 0, params->wsize);-- // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)- {- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;-- for (int64_t i03 = 0; i03 < ne03; i03++) {- for (int64_t i02 = 0; i02 < ne02; i02++) {- const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i03*nb03 + i02*nb02);- ggml_fp16_t * dst_data = wdata + i02*ne01*ne00*ne03;- for (int64_t i01 = 0; i01 < ne01; i01++) {- for (int64_t i00 = 0; i00 < ne00; i00++) {- dst_data[i01*ne00*ne03 + i00*ne03 + i03] = src[i01 * ne00 + i00];- }- }- }- }- }-- // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)- {- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;- for (int i12 = 0; i12 < ne12; i12++) {- for (int i11 = 0; i11 < ne11; i11++) {- const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);- ggml_fp16_t * dst_data = wdata + i11*ne10*ne12;- for (int i10 = 0; i10 < ne10; i10++) {- dst_data[i10*ne12 + i12] = GGML_FP32_TO_FP16(src[i10]);- }- }- }- }+ for (int i12 = 0; i12 < ne12; i12++) {+ for (int i11 = 0; i11 < ne11; i11++) {+ const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);+ ggml_fp16_t * dst_data = wdata + i11*ne10*ne12;+ for (int i10 = 0; i10 < ne10; i10++) {+ dst_data[i10*ne12 + i12] = GGML_FP32_TO_FP16(src[i10]);+ }+ }+ }+ }memset(dst->data, 0, ggml_nbytes(dst));@@ -12460,14 +12125,11 @@ static void ggml_compute_forward_pool_1d(ggml_compute_forward_pool_1d_sk_p0(params, op, src0, k0, dst);}-// ggml_compute_forward_pool_2d_sk_p0+// ggml_compute_forward_pool_2d-static void ggml_compute_forward_pool_2d_sk_p0(+static void ggml_compute_forward_pool_2d(const struct ggml_compute_params * params,- const enum ggml_op_pool op,const struct ggml_tensor * src,- const int k0,- const int k1,struct ggml_tensor * dst) {assert(src->type == GGML_TYPE_F32);assert(params->ith == 0);@@ -12476,6 +12138,14 @@ static void ggml_compute_forward_pool_2d_sk_p0(return;}+ const int32_t * opts = (const int32_t *)dst->op_params;+ enum ggml_op_pool op = opts[0];+ const int k0 = opts[1];+ const int k1 = opts[2];+ const int s0 = opts[3];+ const int s1 = opts[4];+ const int p0 = opts[5];+ const int p1 = opts[6];const char * cdata = (const char*)src->data;const char * const data_end = cdata + ggml_nbytes(src);@@ -12486,6 +12156,8 @@ static void ggml_compute_forward_pool_2d_sk_p0(float * dplane = (float *)dst->data;const int ka = k0 * k1;+ const int offset0 = -p0;+ const int offset1 = -p1;while (cdata < data_end) {for (int oy = 0; oy < py; ++oy) {@@ -12498,13 +12170,15 @@ static void ggml_compute_forward_pool_2d_sk_p0(case GGML_OP_POOL_COUNT: GGML_ASSERT(false); break;}- const int ix = ox * k0;- const int iy = oy * k1;+ const int ix = offset0 + ox * s0;+ const int iy = offset1 + oy * s1;for (int ky = 0; ky < k1; ++ky) {+ if (iy + ky < 0 || iy + ky >= src->ne[1]) continue;const float * const srow = (const float *)(cdata + src->nb[1] * (iy + ky));for (int kx = 0; kx < k0; ++kx) {int j = ix + kx;+ if (j < 0 || j >= src->ne[0]) continue;switch (op) {case GGML_OP_POOL_AVG: *out += srow[j]; break;case GGML_OP_POOL_MAX: if (srow[j] > *out) *out = srow[j]; break;@@ -12525,29 +12199,6 @@ static void ggml_compute_forward_pool_2d_sk_p0(}}-// ggml_compute_forward_pool_2d--static void ggml_compute_forward_pool_2d(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- struct ggml_tensor * dst) {-- const int32_t * opts = (const int32_t *)dst->op_params;- enum ggml_op_pool op = opts[0];- const int k0 = opts[1];- const int k1 = opts[2];- const int s0 = opts[3];- const int s1 = opts[4];- const int p0 = opts[5];- const int p1 = opts[6];- GGML_ASSERT(p0 == 0);- GGML_ASSERT(p1 == 0); // padding not supported- GGML_ASSERT(k0 == s0);- GGML_ASSERT(k1 == s1); // only s = k supported-- ggml_compute_forward_pool_2d_sk_p0(params, op, src0, k0, k1, dst);-}-// ggml_compute_forward_upscalestatic void ggml_compute_forward_upscale_f32(@@ -12603,6 +12254,67 @@ static void ggml_compute_forward_upscale(}}+// ggml_compute_forward_argsort++static void ggml_compute_forward_argsort_f32(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {++ if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {+ return;+ }++ GGML_TENSOR_UNARY_OP_LOCALS++ GGML_ASSERT(nb0 == sizeof(float));++ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t nr = ggml_nrows(src0);++ enum ggml_sort_order order = (enum ggml_sort_order) ggml_get_op_params_i32(dst, 0);++ for (int64_t i = ith; i < nr; i += nth) {+ int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);+ const float * src_data = (float *)((char *) src0->data + i*nb01);++ for (int64_t j = 0; j < ne0; j++) {+ dst_data[j] = j;+ }++ // C doesn't have a functional sort, so we do a bubble sort instead+ for (int64_t j = 0; j < ne0; j++) {+ for (int64_t k = j + 1; k < ne0; k++) {+ if ((order == GGML_SORT_ASC && src_data[dst_data[j]] > src_data[dst_data[k]]) ||+ (order == GGML_SORT_DESC && src_data[dst_data[j]] < src_data[dst_data[k]])) {+ int32_t tmp = dst_data[j];+ dst_data[j] = dst_data[k];+ dst_data[k] = tmp;+ }+ }+ }+ }+}++static void ggml_compute_forward_argsort(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {++ switch (src0->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_argsort_f32(params, src0, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}+// ggml_compute_forward_flash_attnstatic void ggml_compute_forward_flash_attn_f32(@@ -13749,6 +13461,10 @@ static void ggml_compute_forward_unary({ggml_compute_forward_silu(params, src0, dst);} break;+ case GGML_UNARY_OP_LEAKY:+ {+ ggml_compute_forward_leaky(params, src0, dst);+ } break;default:{GGML_ASSERT(false);@@ -14320,12 +14036,14 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggmreturn;}+ if (!ggml_metal_supported() && ggml_cublas_loaded()) {bool skip_cpu = ggml_cuda_compute_forward(params, tensor);if (skip_cpu) {return;}GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU);GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU);+ }switch (tensor->op) {case GGML_OP_DUP:@@ -14418,7 +14136,11 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm} break;case GGML_OP_MUL_MAT:{- ggml_compute_forward_mul_mat(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_mul_mat(params, tensor->src[0], tensor->src[1], tensor, 0, tensor->ne[1]);+ } break;+ case GGML_OP_MUL_MAT_ID:+ {+ ggml_compute_forward_mul_mat_id(params, tensor->src[0], tensor->src[1], tensor);} break;case GGML_OP_OUT_PROD:{@@ -14478,7 +14200,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm} break;case GGML_OP_SOFT_MAX:{- ggml_compute_forward_soft_max(params, tensor->src[0], tensor);+ ggml_compute_forward_soft_max(params, tensor->src[0], tensor->src[1], tensor);} break;case GGML_OP_SOFT_MAX_BACK:{@@ -14500,33 +14222,13 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm{ggml_compute_forward_clamp(params, tensor->src[0], tensor);} break;- case GGML_OP_CONV_1D:- {- ggml_compute_forward_conv_1d(params, tensor->src[0], tensor->src[1], tensor);- } break;- case GGML_OP_CONV_1D_STAGE_0:- {- ggml_compute_forward_conv_1d_stage_0(params, tensor->src[0], tensor->src[1], tensor);- } break;- case GGML_OP_CONV_1D_STAGE_1:- {- ggml_compute_forward_conv_1d_stage_1(params, tensor->src[0], tensor->src[1], tensor);- } break;case GGML_OP_CONV_TRANSPOSE_1D:{ggml_compute_forward_conv_transpose_1d(params, tensor->src[0], tensor->src[1], tensor);} break;- case GGML_OP_CONV_2D:- {- ggml_compute_forward_conv_2d(params, tensor->src[0], tensor->src[1], tensor);- } break;- case GGML_OP_CONV_2D_STAGE_0:- {- ggml_compute_forward_conv_2d_stage_0(params, tensor->src[0], tensor->src[1], tensor);- } break;- case GGML_OP_CONV_2D_STAGE_1:+ case GGML_OP_IM2COL:{- ggml_compute_forward_conv_2d_stage_1(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_im2col(params, tensor->src[0], tensor->src[1], tensor);} break;case GGML_OP_CONV_TRANSPOSE_2D:{@@ -14544,6 +14246,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm{ggml_compute_forward_upscale(params, tensor->src[0], tensor);} break;+ case GGML_OP_ARGSORT:+ {+ ggml_compute_forward_argsort(params, tensor->src[0], tensor);+ } break;case GGML_OP_FLASH_ATTN:{const int32_t t = ggml_get_op_params_i32(tensor, 0);@@ -14655,62 +14361,109 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm////////////////////////////////////////////////////////////////////////////////-static_assert(GGML_GRAPH_HASHTABLE_SIZE > GGML_MAX_NODES * 2, "GGML_GRAPH_HT_SIZE is too small");+static size_t ggml_hash_size(size_t min_sz) {+ // next primes after powers of two+ static const size_t primes[] = {+ 2, 3, 5, 11, 17, 37, 67, 131, 257, 521, 1031,+ 2053, 4099, 8209, 16411, 32771, 65537, 131101,+ 262147, 524309, 1048583, 2097169, 4194319, 8388617,+ 16777259, 33554467, 67108879, 134217757, 268435459,+ 536870923, 1073741827, 2147483659+ };+ static const size_t n_primes = sizeof(primes)/sizeof(primes[0]);++ // find the smallest prime that is larger or equal to min_sz+ size_t l = 0;+ size_t r = n_primes;+ while (l < r) {+ size_t m = (l + r)/2;+ if (primes[m] < min_sz) {+ l = m + 1;+ } else {+ r = m;+ }+ }+ size_t sz = l < n_primes ? primes[l] : min_sz | 1;+ return sz;+}-static size_t hash(void * p) {- return (size_t)p % GGML_GRAPH_HASHTABLE_SIZE;+static size_t ggml_hash(const void * p) {+ return (size_t)p;}-static size_t hash_find(void * hash_table[], void * p) {- size_t h = hash(p);+size_t ggml_hash_find(const struct ggml_hash_set hash_set, struct ggml_tensor * key) {+ size_t h = ggml_hash(key) % hash_set.size;// linear probingsize_t i = h;- while (hash_table[i] != NULL && hash_table[i] != p) {- i = (i + 1) % GGML_GRAPH_HASHTABLE_SIZE;+ while (hash_set.keys[i] != NULL && hash_set.keys[i] != key) {+ i = (i + 1) % hash_set.size;if (i == h) {// visited all hash table entries -> not found- return GGML_GRAPH_HASHTABLE_SIZE;+ return GGML_HASHTABLE_FULL;}}return i;}-static bool hash_insert(void * hash_table[], void * p) {- size_t i = hash_find(hash_table, p);+bool ggml_hash_contains(struct ggml_hash_set hash_set, struct ggml_tensor * key) {+ size_t i = ggml_hash_find(hash_set, key);+ return i != GGML_HASHTABLE_FULL && hash_set.keys[i] == key;+}- GGML_ASSERT(i < GGML_GRAPH_HASHTABLE_SIZE); // assert that not full+size_t ggml_hash_insert(struct ggml_hash_set hash_set, struct ggml_tensor * key) {+ size_t i = ggml_hash_find(hash_set, key);- if (hash_table[i] == p) {- return true;+ GGML_ASSERT(i != GGML_HASHTABLE_FULL);++ if (hash_set.keys[i] == key) {+ return GGML_HASHTABLE_ALREADY_EXISTS;}// insert- GGML_ASSERT(hash_table[i] == NULL);- hash_table[i] = p;- return false;+ GGML_ASSERT(hash_set.keys[i] == NULL);+ hash_set.keys[i] = key;+ return i;+}++size_t ggml_hash_find_or_insert(struct ggml_hash_set hash_set, struct ggml_tensor * key) {+ size_t i = ggml_hash_find(hash_set, key);++ GGML_ASSERT(i != GGML_HASHTABLE_FULL);++ hash_set.keys[i] = key;+ return i;+}++static struct ggml_hash_set ggml_hash_set_new(size_t size) {+ size = ggml_hash_size(size);+ struct ggml_hash_set result;+ result.size = size;+ result.keys = malloc(sizeof(struct ggml_tensor *) * size);+ memset(result.keys, 0, sizeof(struct ggml_tensor *) * size);+ return result;}-static bool hash_contains(void * hash_table[], void * p) {- size_t i = hash_find(hash_table, p);- return (i < GGML_GRAPH_HASHTABLE_SIZE) && (hash_table[i] == p);+static void ggml_hash_set_free(struct ggml_hash_set hash_set) {+ free(hash_set.keys);}struct hash_map {- void * keys[GGML_GRAPH_HASHTABLE_SIZE];- void * vals[GGML_GRAPH_HASHTABLE_SIZE];+ struct ggml_hash_set set;+ struct ggml_tensor ** vals;};-static struct hash_map * new_hash_map(void) {+static struct hash_map * ggml_new_hash_map(size_t size) {struct hash_map * result = malloc(sizeof(struct hash_map));- for (int i=0; i<GGML_GRAPH_HASHTABLE_SIZE; ++i) {- result->keys[i] = NULL;- result->vals[i] = NULL;- }+ result->set = ggml_hash_set_new(size);+ result->vals = malloc(sizeof(struct ggml_tensor *) * result->set.size);+ memset(result->vals, 0, sizeof(struct ggml_tensor *) * result->set.size);return result;}-static void free_hash_map(struct hash_map * map) {+static void ggml_hash_map_free(struct hash_map * map) {+ ggml_hash_set_free(map->set);+ free(map->vals);free(map);}@@ -14730,7 +14483,7 @@ static struct ggml_tensor * ggml_recompute_graph_node(return node;}- if (!hash_contains(graph->visited_hash_table, node)) {+ if (!ggml_hash_contains(graph->visited_hash_table, node)) {return node;}@@ -14745,17 +14498,17 @@ static struct ggml_tensor * ggml_recompute_graph_node(return node;}- size_t i = hash_find(replacements->keys, node);- GGML_ASSERT(i < GGML_GRAPH_HASHTABLE_SIZE); // assert that not full- if (replacements->keys[i] == node) {- return (struct ggml_tensor *) replacements->vals[i];+ size_t i = ggml_hash_find(replacements->set, node);+ GGML_ASSERT(i != GGML_HASHTABLE_FULL); // assert that not full+ if (replacements->set.keys[i] == node) {+ return replacements->vals[i];}struct ggml_tensor * clone = ggml_new_tensor(ctx, node->type, node->n_dims, node->ne);// insert clone into replacements- GGML_ASSERT(replacements->keys[i] == NULL); // assert that we don't overwrite- replacements->keys[i] = node;+ GGML_ASSERT(replacements->set.keys[i] == NULL); // assert that we don't overwrite+ replacements->set.keys[i] = node;replacements->vals[i] = clone;clone->op = node->op;@@ -14792,26 +14545,26 @@ void ggml_build_backward_gradient_checkpointing(struct ggml_cgraph * gb_tmp,struct ggml_tensor * * checkpoints,int n_checkpoints) {- *gb_tmp = *gf;+ ggml_graph_cpy(gf, gb_tmp);ggml_build_backward_expand(ctx, gf, gb_tmp, true);if (n_checkpoints <= 0) {- *gb = *gb_tmp;+ ggml_graph_cpy(gb_tmp, gb);return;}- struct hash_map * replacements = new_hash_map();+ struct hash_map * replacements = ggml_new_hash_map(gf->n_nodes + gf->n_leafs + n_checkpoints);// insert checkpoints in replacementsfor (int i = 0; i < n_checkpoints; ++i) {- size_t k = hash_find(replacements->keys, checkpoints[i]);- GGML_ASSERT(k < GGML_GRAPH_HASHTABLE_SIZE); // assert that not full- GGML_ASSERT(replacements->keys[k] == NULL); // assert that we don't overwrite- replacements->keys[k] = checkpoints[i];- replacements->vals[k] = checkpoints[i];+ size_t k = ggml_hash_find(replacements->set, checkpoints[i]);+ GGML_ASSERT(k != GGML_HASHTABLE_FULL); // assert that not full+ GGML_ASSERT(replacements->set.keys[k] == NULL); // assert that we don't overwrite+ replacements->set.keys[k] = checkpoints[i];+ replacements->vals[k] = checkpoints[i];}- *gb = *gf;+ ggml_graph_cpy(gf, gb);// rewrite gb_tmp->nodes[gf->n_nodes:gb_tmp->n_nodes],// replacing references to gb_tmp->nodes[0:gf->n_nodes] ( == gf->nodes[0:gf->n_nodes]),// by recomputing them from checkpoints@@ -14828,21 +14581,21 @@ void ggml_build_backward_gradient_checkpointing(ggml_build_forward_expand(gb, node);}- free_hash_map(replacements);+ ggml_hash_map_free(replacements);}// functions to change gradients considering the case that input a might be initial gradient with zero value-static struct ggml_tensor * ggml_add_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, void * zero_table[]) {- if (hash_contains(zero_table, a)) {+static struct ggml_tensor * ggml_add_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, struct ggml_hash_set zero_table) {+ if (ggml_hash_contains(zero_table, a)) {return b;} else {return ggml_add_impl(ctx, a, b, false);}}-static struct ggml_tensor * ggml_acc_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, size_t nb1, size_t nb2, size_t nb3, size_t offset, void * zero_table[]) {- if (hash_contains(zero_table, a)) {+static struct ggml_tensor * ggml_acc_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, size_t nb1, size_t nb2, size_t nb3, size_t offset, struct ggml_hash_set zero_table) {+ if (ggml_hash_contains(zero_table, a)) {struct ggml_tensor * a_zero = ggml_scale(ctx, a, ggml_new_f32(ctx, 0));return ggml_acc_impl(ctx, a_zero, b, nb1, nb2, nb3, offset, false);} else {@@ -14850,23 +14603,23 @@ static struct ggml_tensor * ggml_acc_or_set(struct ggml_context * ctx, struct gg}}-static struct ggml_tensor * ggml_add1_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, void * zero_table[]) {- if (hash_contains(zero_table, a)) {+static struct ggml_tensor * ggml_add1_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, struct ggml_hash_set zero_table) {+ if (ggml_hash_contains(zero_table, a)) {return ggml_repeat(ctx, b, a);} else {return ggml_add1_impl(ctx, a, b, false);}}-static struct ggml_tensor * ggml_sub_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, void * zero_table[]) {- if (hash_contains(zero_table, a)) {+static struct ggml_tensor * ggml_sub_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, struct ggml_hash_set zero_table) {+ if (ggml_hash_contains(zero_table, a)) {return ggml_neg(ctx, b);} else {return ggml_sub_impl(ctx, a, b, false);}}-static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor * tensor, void * zero_table[]) {+static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor * tensor, struct ggml_hash_set zero_table) {struct ggml_tensor * src0 = tensor->src[0];struct ggml_tensor * src1 = tensor->src[1];@@ -15147,6 +14900,10 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorzero_table);}} break;+ case GGML_OP_MUL_MAT_ID:+ {+ GGML_ASSERT(false); // TODO: not implemented+ } break;case GGML_OP_OUT_PROD:{GGML_ASSERT(false); // TODO: not implemented@@ -15461,31 +15218,11 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor{GGML_ASSERT(false); // TODO: not implemented} break;- case GGML_OP_CONV_1D:- {- GGML_ASSERT(false); // TODO: not implemented- } break;- case GGML_OP_CONV_1D_STAGE_0:- {- GGML_ASSERT(false); // TODO: not implemented- } break;- case GGML_OP_CONV_1D_STAGE_1:- {- GGML_ASSERT(false); // TODO: not implemented- } break;case GGML_OP_CONV_TRANSPOSE_1D:{GGML_ASSERT(false); // TODO: not implemented} break;- case GGML_OP_CONV_2D:- {- GGML_ASSERT(false); // TODO: not implemented- } break;- case GGML_OP_CONV_2D_STAGE_0:- {- GGML_ASSERT(false); // TODO: not implemented- } break;- case GGML_OP_CONV_2D_STAGE_1:+ case GGML_OP_IM2COL:{GGML_ASSERT(false); // TODO: not implemented} break;@@ -15505,6 +15242,10 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor{GGML_ASSERT(false); // TODO: not implemented} break;+ case GGML_OP_ARGSORT:+ {+ GGML_ASSERT(false); // TODO: not implemented+ } break;case GGML_OP_FLASH_ATTN:{struct ggml_tensor * flash_grad = NULL;@@ -15699,7 +15440,7 @@ static void ggml_visit_parents(struct ggml_cgraph * cgraph, struct ggml_tensor *}// check if already visited- if (hash_insert(cgraph->visited_hash_table, node)) {+ if (ggml_hash_insert(cgraph->visited_hash_table, node) == GGML_HASHTABLE_ALREADY_EXISTS) {return;}@@ -15715,7 +15456,7 @@ static void ggml_visit_parents(struct ggml_cgraph * cgraph, struct ggml_tensor *if (node->op == GGML_OP_NONE && node->grad == NULL) {// reached a leaf node, not part of the gradient graph (e.g. a constant)- GGML_ASSERT(cgraph->n_leafs < GGML_MAX_NODES);+ GGML_ASSERT(cgraph->n_leafs < cgraph->size);if (strlen(node->name) == 0) {ggml_format_name(node, "leaf_%d", cgraph->n_leafs);@@ -15724,22 +15465,24 @@ static void ggml_visit_parents(struct ggml_cgraph * cgraph, struct ggml_tensor *cgraph->leafs[cgraph->n_leafs] = node;cgraph->n_leafs++;} else {- GGML_ASSERT(cgraph->n_nodes < GGML_MAX_NODES);+ GGML_ASSERT(cgraph->n_nodes < cgraph->size);if (strlen(node->name) == 0) {ggml_format_name(node, "node_%d", cgraph->n_nodes);}cgraph->nodes[cgraph->n_nodes] = node;- cgraph->grads[cgraph->n_nodes] = node->grad;+ if (cgraph->grads) {+ cgraph->grads[cgraph->n_nodes] = node->grad;+ }cgraph->n_nodes++;}}static void ggml_build_forward_impl(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor, bool expand) {if (!expand) {- cgraph->n_nodes = 0;- cgraph->n_leafs = 0;+ // TODO: this branch isn't accessible anymore, maybe move this to ggml_build_forward_expand+ ggml_graph_clear(cgraph);}const int n0 = cgraph->n_nodes;@@ -15747,36 +15490,17 @@ static void ggml_build_forward_impl(struct ggml_cgraph * cgraph, struct ggml_tenggml_visit_parents(cgraph, tensor);- const int n_new = cgraph->n_nodes - n0;- GGML_PRINT_DEBUG("%s: visited %d new nodes\n", __func__, n_new);-- if (n_new > 0) {- // the last added node should always be starting point- GGML_ASSERT(cgraph->nodes[cgraph->n_nodes - 1] == tensor);- }-}--void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {- ggml_build_forward_impl(cgraph, tensor, true);-}--struct ggml_cgraph ggml_build_forward(struct ggml_tensor * tensor) {- struct ggml_cgraph result = {- /*.n_nodes =*/ 0,- /*.n_leafs =*/ 0,- /*.nodes =*/ { NULL },- /*.grads =*/ { NULL },- /*.leafs =*/ { NULL },- /*.hash_table =*/ { NULL },- /*.order =*/ GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT,- /*.perf_runs =*/ 0,- /*.perf_cycles =*/ 0,- /*.perf_time_us =*/ 0,- };+ const int n_new = cgraph->n_nodes - n0;+ GGML_PRINT_DEBUG("%s: visited %d new nodes\n", __func__, n_new);- ggml_build_forward_impl(&result, tensor, false);+ if (n_new > 0) {+ // the last added node should always be starting point+ GGML_ASSERT(cgraph->nodes[cgraph->n_nodes - 1] == tensor);+ }+}- return result;+void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {+ ggml_build_forward_impl(cgraph, tensor, true);}void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, bool keep) {@@ -15795,11 +15519,10 @@ void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph *}// remember original gradients which start with zero values- void ** zero_table = malloc(sizeof(void *) * GGML_GRAPH_HASHTABLE_SIZE);- memset(zero_table, 0, sizeof(void*) * GGML_GRAPH_HASHTABLE_SIZE);+ struct ggml_hash_set zero_table = ggml_hash_set_new(gf->size);for (int i = 0; i < gf->n_nodes; i++) {if (gf->grads[i]) {- hash_insert(zero_table, gf->grads[i]);+ ggml_hash_insert(zero_table, gf->grads[i]);}}@@ -15822,26 +15545,54 @@ void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph *}}- free(zero_table);+ ggml_hash_set_free(zero_table);}-struct ggml_cgraph ggml_build_backward(struct ggml_context * ctx, struct ggml_cgraph * gf, bool keep) {- struct ggml_cgraph result = *gf;- ggml_build_backward_expand(ctx, gf, &result, keep);- return result;+static size_t ggml_graph_nbytes(size_t size, bool grads) {+ size_t nbytes = sizeof(struct ggml_cgraph);+ nbytes += size * sizeof(struct ggml_tensor *) * 2; // leafs + nodes+ if (grads) {+ nbytes += size * sizeof(struct ggml_tensor *); // grads+ }+ nbytes += ggml_hash_size(size * 2) * sizeof(struct ggml_tensor *); // hash set+ return nbytes;}-struct ggml_cgraph * ggml_new_graph(struct ggml_context * ctx) {- struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_GRAPH, GGML_GRAPH_SIZE);+size_t ggml_graph_overhead_custom(size_t size, bool grads) {+ return GGML_OBJECT_SIZE + GGML_PAD(ggml_graph_nbytes(size, grads), GGML_MEM_ALIGN);+}++size_t ggml_graph_overhead(void) {+ return ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, false);+}++struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads) {+ const size_t obj_size = ggml_graph_nbytes(size, grads);+ struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_GRAPH, obj_size);struct ggml_cgraph * cgraph = (struct ggml_cgraph *) ((char *) ctx->mem_buffer + obj->offs);+ struct ggml_tensor ** data_start = (struct ggml_tensor **) (cgraph + 1);++ size_t hash_size = ggml_hash_size(size * 2);+ struct ggml_tensor ** nodes_ptr = data_start;+ struct ggml_tensor ** leafs_ptr = nodes_ptr + size;+ struct ggml_tensor ** hash_keys_ptr = leafs_ptr + size;+ struct ggml_tensor ** grads_ptr = grads ? hash_keys_ptr + hash_size : NULL;++ // check that we allocated the correct amount of memory+ assert(obj_size == (size_t) (+ (grads ? (char *)(grads_ptr + size) : (char *)(hash_keys_ptr + hash_size)) - (char *)cgraph));++ memset(hash_keys_ptr, 0, hash_size * sizeof(struct ggml_tensor *));+*cgraph = (struct ggml_cgraph) {+ /*.size =*/ size,/*.n_nodes =*/ 0,/*.n_leafs =*/ 0,- /*.nodes =*/ { NULL },- /*.grads =*/ { NULL },- /*.leafs =*/ { NULL },- /*.hash_table =*/ { NULL },+ /*.nodes =*/ nodes_ptr,+ /*.grads =*/ grads_ptr,+ /*.leafs =*/ leafs_ptr,+ /*.hash_table =*/ { hash_size, hash_keys_ptr },/*.order =*/ GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT,/*.perf_runs =*/ 0,/*.perf_cycles =*/ 0,@@ -15851,14 +15602,81 @@ struct ggml_cgraph * ggml_new_graph(struct ggml_context * ctx) {return cgraph;}-struct ggml_cgraph * ggml_build_forward_ctx(struct ggml_context * ctx, struct ggml_tensor * tensor) {- struct ggml_cgraph * cgraph = ggml_new_graph(ctx);- ggml_build_forward_impl(cgraph, tensor, false);+struct ggml_cgraph * ggml_new_graph(struct ggml_context * ctx) {+ return ggml_new_graph_custom(ctx, GGML_DEFAULT_GRAPH_SIZE, false);+}++struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph0, int i0, int i1) {+ struct ggml_cgraph cgraph = {+ /*.size =*/ 0,+ /*.n_nodes =*/ i1 - i0,+ /*.n_leafs =*/ 0,+ /*.nodes =*/ cgraph0->nodes + i0,+ /*.grads =*/ cgraph0->grads ? cgraph0->grads + i0 : NULL,+ /*.leafs =*/ NULL,+ /*.hash_table =*/ { 0, NULL },+ /*.order =*/ cgraph0->order,+ /*.perf_runs =*/ 0,+ /*.perf_cycles =*/ 0,+ /*.perf_time_us =*/ 0,+ };+return cgraph;}-size_t ggml_graph_overhead(void) {- return GGML_OBJECT_SIZE + GGML_PAD(GGML_GRAPH_SIZE, GGML_MEM_ALIGN);+void ggml_graph_cpy(struct ggml_cgraph * src, struct ggml_cgraph * dst) {+ GGML_ASSERT(dst->size >= src->n_leafs);+ GGML_ASSERT(dst->size >= src->n_nodes);+ GGML_ASSERT(dst->visited_hash_table.size >= src->visited_hash_table.size);++ dst->n_leafs = src->n_leafs;+ dst->n_nodes = src->n_nodes;+ dst->order = src->order;++ for (int i = 0; i < src->n_leafs; ++i) {+ dst->leafs[i] = src->leafs[i];+ }++ for (int i = 0; i < src->n_nodes; ++i) {+ dst->nodes[i] = src->nodes[i];+ }++ if (src->grads) {+ GGML_ASSERT(dst->grads != NULL);+ for (int i = 0; i < src->n_nodes; ++i) {+ dst->grads[i] = src->grads[i];+ }+ }++ for (size_t i = 0; i < src->visited_hash_table.size; ++i) {+ if (src->visited_hash_table.keys[i]) {+ ggml_hash_insert(dst->visited_hash_table, src->visited_hash_table.keys[i]);+ }+ }+}++struct ggml_cgraph * ggml_graph_dup(struct ggml_context * ctx, struct ggml_cgraph * cgraph) {+ struct ggml_cgraph * result = ggml_new_graph_custom(ctx, cgraph->size, cgraph->grads != NULL);+ ggml_graph_cpy(cgraph, result);+ return result;+}++void ggml_graph_reset(struct ggml_cgraph * cgraph) {+ GGML_ASSERT(cgraph->grads != NULL);++ for (int i = 0; i < cgraph->n_nodes; i++) {+ struct ggml_tensor * grad = cgraph->grads[i];++ if (grad) {+ ggml_set_zero(grad);+ }+ }+}++void ggml_graph_clear(struct ggml_cgraph * cgraph) {+ cgraph->n_leafs = 0;+ cgraph->n_nodes = 0;+ memset(cgraph->visited_hash_table.keys, 0, cgraph->visited_hash_table.size * sizeof(struct ggml_tensor *));}//@@ -15868,7 +15686,64 @@ size_t ggml_graph_overhead(void) {// I tried using spin locks, but not sure how to use them correctly - the things I tried were slower than busy loops//+#ifdef __APPLE__++//#include <os/lock.h>+//+//typedef os_unfair_lock ggml_lock_t;+//+//#define ggml_lock_init(x) UNUSED(x)+//#define ggml_lock_destroy(x) UNUSED(x)+//#define ggml_lock_lock os_unfair_lock_lock+//#define ggml_lock_unlock os_unfair_lock_unlock+//+//#define GGML_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT++typedef int ggml_lock_t;++#define ggml_lock_init(x) UNUSED(x)+#define ggml_lock_destroy(x) UNUSED(x)+#define ggml_lock_lock(x) UNUSED(x)+#define ggml_lock_unlock(x) UNUSED(x)++#define GGML_LOCK_INITIALIZER 0++typedef pthread_t ggml_thread_t;++#define ggml_thread_create pthread_create+#define ggml_thread_join pthread_join++#else++//typedef pthread_spinlock_t ggml_lock_t;++//#define ggml_lock_init(x) pthread_spin_init(x, PTHREAD_PROCESS_PRIVATE)+//#define ggml_lock_destroy pthread_spin_destroy+//#define ggml_lock_lock pthread_spin_lock+//#define ggml_lock_unlock pthread_spin_unlock++typedef int ggml_lock_t;++#define ggml_lock_init(x) UNUSED(x)+#define ggml_lock_destroy(x) UNUSED(x)+#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))+#define ggml_lock_lock(x) _mm_pause()+#else+#define ggml_lock_lock(x) UNUSED(x)+#endif+#define ggml_lock_unlock(x) UNUSED(x)++#define GGML_LOCK_INITIALIZER 0++typedef pthread_t ggml_thread_t;++#define ggml_thread_create pthread_create+#define ggml_thread_join pthread_join++#endif+// Android's libc implementation "bionic" does not support setting affinity+#if defined(__linux__) && !defined(__BIONIC__)static void set_numa_thread_affinity(int thread_n, int n_threads) {if (!ggml_is_numa()) {return;@@ -15913,39 +15788,286 @@ static void clear_numa_thread_affinity(void) {strerror(rv));}- CPU_FREE(cpus);-}--struct ggml_compute_state_shared {- const struct ggml_cgraph * cgraph;- const struct ggml_cplan * cplan;-- int64_t perf_node_start_cycles;- int64_t perf_node_start_time_us;-- const int n_threads;-- // synchronization primitives- atomic_int n_active; // num active threads- atomic_int node_n; // active graph node-- bool (*abort_callback)(void * data); // abort ggml_graph_compute when true- void * abort_callback_data;-};--struct ggml_compute_state {- pthread_t thrd;- int ith;- struct ggml_compute_state_shared * shared;-};--static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const struct ggml_compute_state_shared * st) {- int64_t cycles_cur = ggml_perf_cycles() - st->perf_node_start_cycles;- int64_t time_us_cur = ggml_perf_time_us() - st->perf_node_start_time_us;+ CPU_FREE(cpus);+}+#else+// TODO: Windows etc.+// (the linux implementation may also work on BSD, someone should test)+static void set_numa_thread_affinity(int thread_n, int n_threads) { UNUSED(thread_n); UNUSED(n_threads); }+static void clear_numa_thread_affinity(void) {}+#endif++struct ggml_compute_state_shared {+ const struct ggml_cgraph * cgraph;+ const struct ggml_cplan * cplan;++ int64_t perf_node_start_cycles;+ int64_t perf_node_start_time_us;++ const int n_threads;++ // synchronization primitives+ atomic_int n_active; // num active threads+ atomic_int node_n; // active graph node++ bool (*abort_callback)(void * data); // abort ggml_graph_compute when true+ void * abort_callback_data;+};++struct ggml_compute_state {+ ggml_thread_t thrd;+ int ith;+ struct ggml_compute_state_shared * shared;+};++static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const struct ggml_compute_state_shared * st) {+ int64_t cycles_cur = ggml_perf_cycles() - st->perf_node_start_cycles;+ int64_t time_us_cur = ggml_perf_time_us() - st->perf_node_start_time_us;++ node->perf_runs++;+ node->perf_cycles += cycles_cur;+ node->perf_time_us += time_us_cur;+}++static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {+ int n_tasks = 0;++ switch (node->op) {+ case GGML_OP_CPY:+ case GGML_OP_DUP:+ case GGML_OP_ADD:+ case GGML_OP_ADD1:+ case GGML_OP_ACC:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_SUB:+ case GGML_OP_SQR:+ case GGML_OP_SQRT:+ case GGML_OP_LOG:+ case GGML_OP_SUM:+ case GGML_OP_SUM_ROWS:+ case GGML_OP_MEAN:+ case GGML_OP_ARGMAX:+ case GGML_OP_REPEAT:+ case GGML_OP_REPEAT_BACK:+ {+ n_tasks = 1;+ } break;+ case GGML_OP_UNARY:+ switch (ggml_get_unary_op(node)) {+ case GGML_UNARY_OP_ABS:+ case GGML_UNARY_OP_SGN:+ case GGML_UNARY_OP_NEG:+ case GGML_UNARY_OP_STEP:+ case GGML_UNARY_OP_TANH:+ case GGML_UNARY_OP_ELU:+ case GGML_UNARY_OP_RELU:+ case GGML_UNARY_OP_LEAKY:+ {+ n_tasks = 1;+ } break;++ case GGML_UNARY_OP_GELU:+ case GGML_UNARY_OP_GELU_QUICK:+ case GGML_UNARY_OP_SILU:+ {+ n_tasks = n_threads;+ } break;+ default:+ GGML_ASSERT(false);+ }+ break;+ case GGML_OP_SILU_BACK:+ case GGML_OP_MUL:+ case GGML_OP_DIV:+ case GGML_OP_NORM:+ case GGML_OP_RMS_NORM:+ case GGML_OP_RMS_NORM_BACK:+ case GGML_OP_GROUP_NORM:+ case GGML_OP_CONCAT:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_MUL_MAT:+ {+ n_tasks = n_threads;++ // TODO: use different scheduling for different matrix sizes+ //const int nr0 = ggml_nrows(node->src[0]);+ //const int nr1 = ggml_nrows(node->src[1]);++ //n_tasks = MIN(n_threads, MAX(1, nr0/128));+ //printf("nr0 = %8d, nr1 = %8d, nr0*nr1 = %8d, n_tasks%d\n", nr0, nr1, nr0*nr1, n_tasks);++ if (!ggml_metal_supported() && ggml_cublas_loaded()) {+ if (ggml_cuda_can_mul_mat(node->src[0], node->src[1], node)) {+ n_tasks = 1; // TODO: this actually is doing nothing+ // the threads are still spinning+ }+ }+#if defined(GGML_USE_CLBLAST)+ if (ggml_cl_can_mul_mat(node->src[0], node->src[1], node)) {+ n_tasks = 1; // TODO: this actually is doing nothing+ // the threads are still spinning+ }+#endif+#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)+ if (ggml_compute_forward_mul_mat_use_blas(node->src[0], node->src[1], node)) {+ n_tasks = 1; // TODO: this actually is doing nothing+ // the threads are still spinning+ }+#endif+ } break;+ case GGML_OP_MUL_MAT_ID:+ {+ // FIXME: blas+ n_tasks = n_threads;+ } break;+ case GGML_OP_OUT_PROD:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_SCALE:+ case GGML_OP_SET:+ case GGML_OP_CONT:+ case GGML_OP_RESHAPE:+ case GGML_OP_VIEW:+ case GGML_OP_PERMUTE:+ case GGML_OP_TRANSPOSE:+ case GGML_OP_GET_ROWS:+ case GGML_OP_GET_ROWS_BACK:+ case GGML_OP_DIAG:+ {+ n_tasks = 1;+ } break;+ case GGML_OP_DIAG_MASK_ZERO:+ case GGML_OP_DIAG_MASK_INF:+ case GGML_OP_SOFT_MAX_BACK:+ case GGML_OP_ROPE:+ case GGML_OP_ROPE_BACK:+ case GGML_OP_ADD_REL_POS:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_ALIBI:+ {+ n_tasks = 1; //TODO+ } break;+ case GGML_OP_CLAMP:+ {+ n_tasks = 1; //TODO+ } break;+ case GGML_OP_SOFT_MAX:+ {+ n_tasks = MIN(MIN(4, n_threads), ggml_nrows(node->src[0]));+ } break;+ case GGML_OP_CONV_TRANSPOSE_1D:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_IM2COL:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_CONV_TRANSPOSE_2D:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_POOL_1D:+ case GGML_OP_POOL_2D:+ {+ n_tasks = 1;+ } break;+ case GGML_OP_UPSCALE:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_ARGSORT:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_FLASH_ATTN:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_FLASH_FF:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_FLASH_ATTN_BACK:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_WIN_PART:+ case GGML_OP_WIN_UNPART:+ case GGML_OP_GET_REL_POS:+ case GGML_OP_MAP_UNARY:+ case GGML_OP_MAP_BINARY:+ case GGML_OP_MAP_CUSTOM1_F32:+ case GGML_OP_MAP_CUSTOM2_F32:+ case GGML_OP_MAP_CUSTOM3_F32:+ {+ n_tasks = 1;+ } break;+ case GGML_OP_MAP_CUSTOM1:+ {+ struct ggml_map_custom1_op_params * p = (struct ggml_map_custom1_op_params *) node->op_params;+ if (p->n_tasks == GGML_N_TASKS_MAX) {+ n_tasks = n_threads;+ } else {+ n_tasks = MIN(p->n_tasks, n_threads);+ }+ } break;+ case GGML_OP_MAP_CUSTOM2:+ {+ struct ggml_map_custom2_op_params * p = (struct ggml_map_custom2_op_params *) node->op_params;+ if (p->n_tasks == GGML_N_TASKS_MAX) {+ n_tasks = n_threads;+ } else {+ n_tasks = MIN(p->n_tasks, n_threads);+ }+ } break;+ case GGML_OP_MAP_CUSTOM3:+ {+ struct ggml_map_custom3_op_params * p = (struct ggml_map_custom3_op_params *) node->op_params;+ if (p->n_tasks == GGML_N_TASKS_MAX) {+ n_tasks = n_threads;+ } else {+ n_tasks = MIN(p->n_tasks, n_threads);+ }+ } break;+ case GGML_OP_CROSS_ENTROPY_LOSS:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_CROSS_ENTROPY_LOSS_BACK:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_NONE:+ {+ n_tasks = 1;+ } break;+ case GGML_OP_COUNT:+ {+ GGML_ASSERT(false);+ } break;+ default:+ {+ fprintf(stderr, "%s: op not implemented: ", __func__);+ if (node->op < GGML_OP_COUNT) {+ fprintf(stderr, "%s\n", ggml_op_name(node->op));+ } else {+ fprintf(stderr, "%d\n", node->op);+ }+ GGML_ASSERT(false);+ } break;+ }++ assert(n_tasks > 0);- node->perf_runs++;- node->perf_cycles += cycles_cur;- node->perf_time_us += time_us_cur;+ return n_tasks;}static thread_ret_t ggml_graph_compute_thread(void * data) {@@ -15954,7 +16076,6 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {const struct ggml_cgraph * cgraph = state->shared->cgraph;const struct ggml_cplan * cplan = state->shared->cplan;- const int * n_tasks_arr = cplan->n_tasks;const int n_threads = state->shared->n_threads;set_numa_thread_affinity(state->ith, n_threads);@@ -15979,9 +16100,9 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {if (node_n != -1) {/* FINALIZE */- struct ggml_tensor * node = state->shared->cgraph->nodes[node_n];+ struct ggml_tensor * node = cgraph->nodes[node_n];if (GGML_OP_HAS_FINALIZE[node->op]) {- params.nth = n_tasks_arr[node_n];+ params.nth = ggml_get_n_tasks(node, n_threads);ggml_compute_forward(¶ms, node);}ggml_graph_compute_perf_stats_node(node, state->shared);@@ -15992,7 +16113,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {GGML_PRINT_DEBUG_5("%s: %d/%d\n", __func__, node_n, cgraph->n_nodes);struct ggml_tensor * node = cgraph->nodes[node_n];- const int n_tasks = n_tasks_arr[node_n];+ const int n_tasks = ggml_get_n_tasks(node, n_threads);state->shared->perf_node_start_cycles = ggml_perf_cycles();state->shared->perf_node_start_time_us = ggml_perf_time_us();@@ -16050,7 +16171,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {/* COMPUTE */struct ggml_tensor * node = cgraph->nodes[node_n];- const int n_tasks = n_tasks_arr[node_n];+ const int n_tasks = ggml_get_n_tasks(node, n_threads);struct ggml_compute_params params = {/*.type =*/ GGML_TASK_COMPUTE,@@ -16080,125 +16201,44 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {// thread scheduling for the different operations + work buffer size estimationfor (int i = 0; i < cgraph->n_nodes; i++) {- int n_tasks = 1;-struct ggml_tensor * node = cgraph->nodes[i];+ const int n_tasks = ggml_get_n_tasks(node, n_threads);++ size_t cur = 0;+switch (node->op) {case GGML_OP_CPY:case GGML_OP_DUP:{- n_tasks = n_threads;-- size_t cur = 0;if (ggml_is_quantized(node->type)) {cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;}-- work_size = MAX(work_size, cur);} break;case GGML_OP_ADD:case GGML_OP_ADD1:{- n_tasks = n_threads;-- size_t cur = 0;-if (ggml_is_quantized(node->src[0]->type)) {cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;}-- work_size = MAX(work_size, cur);} break;case GGML_OP_ACC:{- n_tasks = n_threads;-- size_t cur = 0;-if (ggml_is_quantized(node->src[0]->type)) {cur = ggml_type_size(GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks;}-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_SUB:- case GGML_OP_DIV:- case GGML_OP_SQR:- case GGML_OP_SQRT:- case GGML_OP_LOG:- case GGML_OP_SUM:- case GGML_OP_SUM_ROWS:- case GGML_OP_MEAN:- case GGML_OP_ARGMAX:- case GGML_OP_REPEAT:- case GGML_OP_REPEAT_BACK:- {- n_tasks = 1;- } break;-- case GGML_OP_UNARY:- {- switch (ggml_get_unary_op(node)) {- case GGML_UNARY_OP_ABS:- case GGML_UNARY_OP_SGN:- case GGML_UNARY_OP_NEG:- case GGML_UNARY_OP_STEP:- case GGML_UNARY_OP_TANH:- case GGML_UNARY_OP_ELU:- case GGML_UNARY_OP_RELU:- {- n_tasks = 1;- } break;-- case GGML_UNARY_OP_GELU:- case GGML_UNARY_OP_GELU_QUICK:- case GGML_UNARY_OP_SILU:- {- n_tasks = n_threads;- } break;- }- } break;- case GGML_OP_SILU_BACK:- case GGML_OP_MUL:- case GGML_OP_NORM:- case GGML_OP_RMS_NORM:- case GGML_OP_RMS_NORM_BACK:- case GGML_OP_GROUP_NORM:- {- n_tasks = n_threads;} break;- case GGML_OP_CONCAT:case GGML_OP_MUL_MAT:{- n_tasks = n_threads;-- // TODO: use different scheduling for different matrix sizes- //const int nr0 = ggml_nrows(node->src[0]);- //const int nr1 = ggml_nrows(node->src[1]);-- //n_tasks = MIN(n_threads, MAX(1, nr0/128));- //printf("nr0 = %8d, nr1 = %8d, nr0*nr1 = %8d, n_tasks%d\n", nr0, nr1, nr0*nr1, n_tasks);-- size_t cur = 0;const enum ggml_type vec_dot_type = type_traits[node->src[0]->type].vec_dot_type;-#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_TINYBLAS)- if (ggml_cuda_can_mul_mat(node->src[0], node->src[1], node)) {- n_tasks = 1; // TODO: this actually is doing nothing- // the threads are still spinning- } else-#elif defined(GGML_USE_CLBLAST)+#if defined(GGML_USE_CLBLAST)if (ggml_cl_can_mul_mat(node->src[0], node->src[1], node)) {- n_tasks = 1; // TODO: this actually is doing nothing- // the threads are still spinningcur = ggml_cl_mul_mat_get_wsize(node->src[0], node->src[1], node);} else#endif#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)if (ggml_compute_forward_mul_mat_use_blas(node->src[0], node->src[1], node)) {- n_tasks = 1; // TODO: this actually is doing nothing- // the threads are still spinningif (node->src[0]->type != GGML_TYPE_F32) {// here we need memory just for single 2D matrix from src0cur = ggml_type_size(GGML_TYPE_F32)*(node->src[0]->ne[0]*node->src[0]->ne[1]);@@ -16207,108 +16247,37 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {#endifif (node->src[1]->type != vec_dot_type) {cur = ggml_type_size(vec_dot_type)*ggml_nelements(node->src[1])/ggml_blck_size(vec_dot_type);- } else {- cur = 0;}-- work_size = MAX(work_size, cur);+ } break;+ case GGML_OP_MUL_MAT_ID:+ {+ const struct ggml_tensor * a = node->src[2];+ const struct ggml_tensor * b = node->src[1];+ const enum ggml_type vec_dot_type = type_traits[a->type].vec_dot_type;+#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)+ if (ggml_compute_forward_mul_mat_use_blas(a, b, node)) {+ if (a->type != GGML_TYPE_F32) {+ // here we need memory just for single 2D matrix from src0+ cur = ggml_type_size(GGML_TYPE_F32)*(a->ne[0]*a->ne[1]);+ }+ } else+#endif+ if (b->type != vec_dot_type) {+ cur = ggml_type_size(vec_dot_type)*ggml_nelements(b)/ggml_blck_size(vec_dot_type);+ }} break;case GGML_OP_OUT_PROD:{- n_tasks = n_threads;-- size_t cur = 0;-if (ggml_is_quantized(node->src[0]->type)) {cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;}-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_SCALE:- {- n_tasks = 1;} break;- case GGML_OP_SET:- case GGML_OP_CONT:- case GGML_OP_RESHAPE:- case GGML_OP_VIEW:- case GGML_OP_PERMUTE:- case GGML_OP_TRANSPOSE:- case GGML_OP_GET_ROWS:- case GGML_OP_GET_ROWS_BACK:- case GGML_OP_DIAG:- {- n_tasks = 1;- } break;- case GGML_OP_DIAG_MASK_ZERO:- case GGML_OP_DIAG_MASK_INF:case GGML_OP_SOFT_MAX:- case GGML_OP_SOFT_MAX_BACK:- case GGML_OP_ROPE:- case GGML_OP_ROPE_BACK:- case GGML_OP_ADD_REL_POS:- {- n_tasks = n_threads;- } break;- case GGML_OP_ALIBI:- {- n_tasks = 1; //TODO- } break;- case GGML_OP_CLAMP:- {- n_tasks = 1; //TODO- } break;- case GGML_OP_CONV_1D:- {- n_tasks = n_threads;-- GGML_ASSERT(node->src[0]->ne[3] == 1);- GGML_ASSERT(node->src[1]->ne[2] == 1);- GGML_ASSERT(node->src[1]->ne[3] == 1);-- const int64_t ne00 = node->src[0]->ne[0];- const int64_t ne01 = node->src[0]->ne[1];- const int64_t ne02 = node->src[0]->ne[2];-- const int64_t ne10 = node->src[1]->ne[0];- const int64_t ne11 = node->src[1]->ne[1];-- const int64_t ne0 = node->ne[0];- const int64_t ne1 = node->ne[1];- const int64_t nk = ne00;- const int64_t ew0 = nk * ne01;-- UNUSED(ne02);- UNUSED(ne10);- UNUSED(ne11);-- size_t cur = 0;-- if (node->src[0]->type == GGML_TYPE_F16 &&- node->src[1]->type == GGML_TYPE_F32) {- cur = sizeof(ggml_fp16_t)*(ne0*ne1*ew0);- } else if (node->src[0]->type == GGML_TYPE_F32 &&- node->src[1]->type == GGML_TYPE_F32) {- cur = sizeof(float)*(ne0*ne1*ew0);- } else {- GGML_ASSERT(false);- }-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_CONV_1D_STAGE_0:- {- n_tasks = n_threads;- } break;- case GGML_OP_CONV_1D_STAGE_1:{- n_tasks = n_threads;+ cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;} break;case GGML_OP_CONV_TRANSPOSE_1D:{- n_tasks = n_threads;-GGML_ASSERT(node->src[0]->ne[3] == 1);GGML_ASSERT(node->src[1]->ne[2] == 1);GGML_ASSERT(node->src[1]->ne[3] == 1);@@ -16320,7 +16289,6 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {const int64_t ne10 = node->src[1]->ne[0]; // Lconst int64_t ne11 = node->src[1]->ne[1]; // Cin- size_t cur = 0;if (node->src[0]->type == GGML_TYPE_F16 &&node->src[1]->type == GGML_TYPE_F32) {cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;@@ -16332,59 +16300,9 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {} else {GGML_ASSERT(false);}-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_CONV_2D:- {- n_tasks = n_threads;-- const int64_t ne00 = node->src[0]->ne[0]; // W- const int64_t ne01 = node->src[0]->ne[1]; // H- const int64_t ne02 = node->src[0]->ne[2]; // C- const int64_t ne03 = node->src[0]->ne[3]; // N-- const int64_t ne10 = node->src[1]->ne[0]; // W- const int64_t ne11 = node->src[1]->ne[1]; // H- const int64_t ne12 = node->src[1]->ne[2]; // C-- const int64_t ne0 = node->ne[0];- const int64_t ne1 = node->ne[1];- const int64_t ne2 = node->ne[2];- const int64_t ne3 = node->ne[3];- const int64_t nk = ne00*ne01;- const int64_t ew0 = nk * ne02;-- UNUSED(ne03);- UNUSED(ne2);-- size_t cur = 0;-- if (node->src[0]->type == GGML_TYPE_F16 &&- node->src[1]->type == GGML_TYPE_F32) {- // im2col: [N*OH*OW, IC*KH*KW]- cur = sizeof(ggml_fp16_t)*(ne3*ne0*ne1*ew0);- } else if (node->src[0]->type == GGML_TYPE_F32 &&- node->src[1]->type == GGML_TYPE_F32) {- cur = sizeof(float)* (ne10*ne11*ne12);- } else {- GGML_ASSERT(false);- }-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_CONV_2D_STAGE_0:- {- n_tasks = n_threads;- } break;- case GGML_OP_CONV_2D_STAGE_1:- {- n_tasks = n_threads;} break;case GGML_OP_CONV_TRANSPOSE_2D:{- n_tasks = n_threads;-const int64_t ne00 = node->src[0]->ne[0]; // Wconst int64_t ne01 = node->src[0]->ne[1]; // Hconst int64_t ne02 = node->src[0]->ne[2]; // Channels Out@@ -16394,141 +16312,58 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {const int64_t ne11 = node->src[1]->ne[1]; // Hconst int64_t ne12 = node->src[1]->ne[2]; // Channels In- size_t cur = 0;cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02*ne03;cur += sizeof(ggml_fp16_t)*ne10*ne11*ne12;-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_POOL_1D:- case GGML_OP_POOL_2D:- {- n_tasks = 1;- } break;- case GGML_OP_UPSCALE:- {- n_tasks = n_threads;} break;case GGML_OP_FLASH_ATTN:{- n_tasks = n_threads;-- size_t cur = 0;-const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL);if (node->src[1]->type == GGML_TYPE_F32) {cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2- }-- if (node->src[1]->type == GGML_TYPE_F16) {+ } else if (node->src[1]->type == GGML_TYPE_F16) {cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2}-- work_size = MAX(work_size, cur);} break;case GGML_OP_FLASH_FF:{- n_tasks = n_threads;-- size_t cur = 0;-if (node->src[1]->type == GGML_TYPE_F32) {cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2- }-- if (node->src[1]->type == GGML_TYPE_F16) {+ } else if (node->src[1]->type == GGML_TYPE_F16) {cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2}-- work_size = MAX(work_size, cur);} break;case GGML_OP_FLASH_ATTN_BACK:{- n_tasks = n_threads;-- size_t cur = 0;-const int64_t D = node->src[0]->ne[0];const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL);const int64_t mxDn = MAX(D, ne11) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_backif (node->src[1]->type == GGML_TYPE_F32) {cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2- }-- if (node->src[1]->type == GGML_TYPE_F16) {+ } else if (node->src[1]->type == GGML_TYPE_F16) {cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2}-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_WIN_PART:- case GGML_OP_WIN_UNPART:- case GGML_OP_GET_REL_POS:- case GGML_OP_MAP_UNARY:- case GGML_OP_MAP_BINARY:- case GGML_OP_MAP_CUSTOM1_F32:- case GGML_OP_MAP_CUSTOM2_F32:- case GGML_OP_MAP_CUSTOM3_F32:- {- n_tasks = 1;- } break;- case GGML_OP_MAP_CUSTOM1:- {- struct ggml_map_custom1_op_params * p = (struct ggml_map_custom1_op_params *) node->op_params;- if (p->n_tasks == GGML_N_TASKS_MAX) {- n_tasks = n_threads;- } else {- n_tasks = MIN(p->n_tasks, n_threads);- }- } break;- case GGML_OP_MAP_CUSTOM2:- {- struct ggml_map_custom2_op_params * p = (struct ggml_map_custom2_op_params *) node->op_params;- if (p->n_tasks == GGML_N_TASKS_MAX) {- n_tasks = n_threads;- } else {- n_tasks = MIN(p->n_tasks, n_threads);- }- } break;- case GGML_OP_MAP_CUSTOM3:- {- struct ggml_map_custom3_op_params * p = (struct ggml_map_custom3_op_params *) node->op_params;- if (p->n_tasks == GGML_N_TASKS_MAX) {- n_tasks = n_threads;- } else {- n_tasks = MIN(p->n_tasks, n_threads);- }} break;- case GGML_OP_CROSS_ENTROPY_LOSS:- {- n_tasks = n_threads;- size_t cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);-- work_size = MAX(work_size, cur);- } break;- case GGML_OP_CROSS_ENTROPY_LOSS_BACK:- {- n_tasks = n_threads;- } break;- case GGML_OP_NONE:+ case GGML_OP_CROSS_ENTROPY_LOSS:{- n_tasks = 1;+ cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);} break;case GGML_OP_COUNT:{GGML_ASSERT(false);} break;+ default:+ break;}- cplan.n_tasks[i] = n_tasks;+ work_size = MAX(work_size, cur);}if (work_size > 0) {@@ -16550,12 +16385,6 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {if (cplan->work_size > 0) {GGML_ASSERT(cplan->work_data);}-- for (int i = 0; i < cgraph->n_nodes; ++i) {- if (cgraph->nodes[i]->op != GGML_OP_NONE) {- GGML_ASSERT(cplan->n_tasks[i] > 0);- }- }}const int n_threads = cplan->n_threads;@@ -16582,7 +16411,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {.shared = &state_shared,};- const int rc = pthread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);+ const int rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);GGML_ASSERT(rc == 0);UNUSED(rc);}@@ -16603,7 +16432,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {// join or kill thread poolif (n_threads > 1) {for (int j = 1; j < n_threads; j++) {- const int rc = pthread_join(workers[j].thrd, NULL);+ const int rc = ggml_thread_join(workers[j].thrd, NULL);GGML_ASSERT(rc == 0);}}@@ -16628,16 +16457,6 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {return compute_status;}-void ggml_graph_reset(struct ggml_cgraph * cgraph) {- for (int i = 0; i < cgraph->n_nodes; i++) {- struct ggml_tensor * grad = cgraph->grads[i];-- if (grad) {- ggml_set_zero(grad);- }- }-}-void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads);@@ -16764,12 +16583,12 @@ void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) {const uint32_t magic = GGML_FILE_MAGIC;const uint32_t version = GGML_FILE_VERSION;const uint32_t n_leafs = cgraph->n_leafs;- const uint32_t nodes = cgraph->n_nodes;+ const uint32_t n_nodes = cgraph->n_nodes;fwrite(&magic, sizeof(uint32_t), 1, fout);fwrite(&version, sizeof(uint32_t), 1, fout);fwrite(&n_leafs, sizeof(uint32_t), 1, fout);- fwrite(&nodes, sizeof(uint32_t), 1, fout);+ fwrite(&n_nodes, sizeof(uint32_t), 1, fout);fwrite(&size_eval, sizeof(uint64_t), 1, fout);}@@ -16857,7 +16676,7 @@ void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) {if (idx == -1) {for (int k = 0; k < cgraph->n_nodes; ++k) {if (args[j] == cgraph->nodes[k]) {- idx = GGML_MAX_NODES + k;+ idx = cgraph->n_leafs + k;break;}}@@ -16884,11 +16703,11 @@ void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) {}}-struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval) {+struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval) {assert(*ctx_data == NULL);assert(*ctx_eval == NULL);- struct ggml_cgraph result = { 0 };+ struct ggml_cgraph * result = NULL;struct ggml_tensor * data = NULL;@@ -16960,13 +16779,11 @@ struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context **const uint32_t n_leafs = *(const uint32_t *) ptr; ptr += sizeof(n_leafs);const uint32_t n_nodes = *(const uint32_t *) ptr; ptr += sizeof(n_nodes);const uint64_t size_eval = *(const uint64_t *) ptr; ptr += sizeof(size_eval);-- result.n_leafs = n_leafs;- result.n_nodes = n_nodes;+ const int graph_size = MAX(n_leafs, n_nodes);// create the data context{- const size_t overhead = (n_leafs + n_nodes)*ggml_tensor_overhead();+ const size_t overhead = (n_leafs + n_nodes)*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false);struct ggml_init_params params = {.mem_size = size_eval + overhead,@@ -16982,6 +16799,12 @@ struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context **}}+ result = ggml_new_graph_custom(*ctx_eval, graph_size, false);++ result->n_leafs = n_leafs;+ result->n_nodes = n_nodes;++// leafs{uint32_t type;@@ -17020,7 +16843,7 @@ struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context **tensor->nb[j] = nb[j];}- result.leafs[i] = tensor;+ result->leafs[i] = tensor;ptr += ggml_nbytes(tensor);@@ -17072,10 +16895,10 @@ struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context **continue;}- if (arg_idx < GGML_MAX_NODES) {- args[j] = result.leafs[arg_idx];+ if (arg_idx < result->n_leafs) {+ args[j] = result->leafs[arg_idx];} else {- args[j] = result.nodes[arg_idx - GGML_MAX_NODES];+ args[j] = result->nodes[arg_idx - result->n_leafs];}}@@ -17127,7 +16950,7 @@ struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context **tensor->src[j] = args[j];}- result.nodes[i] = tensor;+ result->nodes[i] = tensor;fprintf(stderr, "%s: loaded node %d: '%16s', %3d dims, %9zu bytes\n", __func__, i, tensor->name, n_dims, ggml_nbytes(tensor));}@@ -18032,10 +17855,11 @@ struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) {case GGML_OPT_ADAM:{result = (struct ggml_opt_params) {- .type = GGML_OPT_ADAM,- .n_threads = 1,- .past = 0,- .delta = 1e-5f,+ .type = GGML_OPT_ADAM,+ .graph_size = GGML_DEFAULT_GRAPH_SIZE,+ .n_threads = 1, // FIXME: GGML_DEFAULT_N_THREADS ?+ .past = 0,+ .delta = 1e-5f,.max_no_improvement = 100,@@ -18062,10 +17886,11 @@ struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) {case GGML_OPT_LBFGS:{result = (struct ggml_opt_params) {- .type = GGML_OPT_LBFGS,- .n_threads = 1,- .past = 0,- .delta = 1e-5f,+ .type = GGML_OPT_LBFGS,+ .graph_size = GGML_DEFAULT_GRAPH_SIZE,+ .n_threads = 1,+ .past = 0,+ .delta = 1e-5f,.max_no_improvement = 0,@@ -18207,14 +18032,11 @@ enum ggml_opt_result ggml_opt_resume(struct ggml_tensor * f) {// build forward + backward compute graphs- struct ggml_tensor * gfbuf = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sizeof(struct ggml_cgraph) / ggml_type_size(GGML_TYPE_I32)+ (sizeof(struct ggml_cgraph) % ggml_type_size(GGML_TYPE_I32) ? 1 : 0));- struct ggml_tensor * gbbuf = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sizeof(struct ggml_cgraph) / ggml_type_size(GGML_TYPE_I32)+ (sizeof(struct ggml_cgraph) % ggml_type_size(GGML_TYPE_I32) ? 1 : 0));-- struct ggml_cgraph * gf = (struct ggml_cgraph *) gfbuf->data;- struct ggml_cgraph * gb = (struct ggml_cgraph *) gbbuf->data;+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, opt->params.graph_size, true);+ ggml_build_forward_expand(gf, f);- *gf = ggml_build_forward (f);- *gb = ggml_build_backward(ctx, gf, true);+ struct ggml_cgraph * gb = ggml_graph_dup(ctx, gf);+ ggml_build_backward_expand(ctx, gf, gb, true);return ggml_opt_resume_g(ctx, opt, f, gf, gb, NULL, NULL);}@@ -18317,8 +18139,8 @@ size_t ggml_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t *memcpy(&qh, &y[i].qh, sizeof(qh));for (int j = 0; j < QK5_0; j += 2) {- const uint8_t vh0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;- const uint8_t vh1 = ((qh & (1u << (j + 16))) >> (j + 12));+ const uint8_t vh0 = ((qh & (1u << (j/2 + 0 ))) >> (j/2 + 0 )) << 4;+ const uint8_t vh1 = ((qh & (1u << (j/2 + 16))) >> (j/2 + 12));// cast to 16 binsconst uint8_t vi0 = ((y[i].qs[j/2] & 0x0F) | vh0) / 2;@@ -18347,8 +18169,8 @@ size_t ggml_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t *memcpy(&qh, &y[i].qh, sizeof(qh));for (int j = 0; j < QK5_1; j += 2) {- const uint8_t vh0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;- const uint8_t vh1 = ((qh & (1u << (j + 16))) >> (j + 12));+ const uint8_t vh0 = ((qh & (1u << (j/2 + 0 ))) >> (j/2 + 0 )) << 4;+ const uint8_t vh1 = ((qh & (1u << (j/2 + 16))) >> (j/2 + 12));// cast to 16 binsconst uint8_t vi0 = ((y[i].qs[j/2] & 0x0F) | vh0) / 2;@@ -18538,6 +18360,7 @@ struct gguf_kv {struct gguf_header {char magic[4];+uint32_t version;uint64_t n_tensors; // GGUFv2uint64_t n_kv; // GGUFv2@@ -18578,7 +18401,6 @@ static bool gguf_fread_el(struct llamafile * file, void * dst, size_t size, sizereturn n == size;}-// NOTE: temporary handling of GGUFv1 >> remove after Oct 2023static bool gguf_fread_str(struct llamafile * file, struct gguf_str * p, size_t * offset) {p->n = 0;p->data = NULL;@@ -18592,7 +18414,7 @@ static bool gguf_fread_str(struct llamafile * file, struct gguf_str * p, size_t}struct gguf_context * gguf_init_empty(void) {- struct gguf_context * ctx = ggml_aligned_malloc(sizeof(struct gguf_context));+ struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));memcpy(ctx->header.magic, GGUF_MAGIC, sizeof(ctx->header.magic));ctx->header.version = GGUF_VERSION;@@ -18636,13 +18458,12 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_ibool ok = true;- struct gguf_context * ctx = ggml_aligned_malloc(sizeof(struct gguf_context));+ struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));// read the header{strncpy(ctx->header.magic, magic, 4);-ctx->kv = NULL;ctx->infos = NULL;ctx->data = NULL;@@ -18653,7 +18474,6 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_iif (ctx->header.version == 1) {fprintf(stderr, "%s: GGUFv1 is no longer supported. please use a more up-to-date version\n", __func__);- llamafile_close(file);gguf_free(ctx);return NULL;}@@ -18669,7 +18489,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i{ctx->kv = malloc(ctx->header.n_kv * sizeof(struct gguf_kv));- for (uint32_t i = 0; i < ctx->header.n_kv; ++i) {+ for (uint64_t i = 0; i < ctx->header.n_kv; ++i) {struct gguf_kv * kv = &ctx->kv[i];//fprintf(stderr, "%s: reading kv %d\n", __func__, i);@@ -18716,7 +18536,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_icase GGUF_TYPE_STRING:{kv->value.arr.data = malloc(kv->value.arr.n * sizeof(struct gguf_str));- for (uint32_t j = 0; j < kv->value.arr.n; ++j) {+ for (uint64_t j = 0; j < kv->value.arr.n; ++j) {ok = ok && gguf_fread_str(file, &((struct gguf_str *) kv->value.arr.data)[j], &offset);}} break;@@ -18743,7 +18563,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i{ctx->infos = malloc(ctx->header.n_tensors * sizeof(struct gguf_tensor_info));- for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) {+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {struct gguf_tensor_info * info = &ctx->infos[i];for (int j = 0; j < GGML_MAX_DIMS; ++j) {@@ -18789,7 +18609,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i// compute the total size of the data section, taking into account the alignment{ctx->size = 0;- for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) {+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {struct gguf_tensor_info * info = &ctx->infos[i];const int64_t ne =@@ -18856,7 +18676,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_iggml_set_no_alloc(ctx_data, true);// create the tensors- for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) {+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {const int64_t ne[GGML_MAX_DIMS] = {ctx->infos[i].ne[0],ctx->infos[i].ne[1],@@ -18944,7 +18764,7 @@ void gguf_free(struct gguf_context * ctx) {free(ctx->infos);}- ggml_aligned_free(ctx);+ GGML_ALIGNED_FREE(ctx);}const char * gguf_type_name(enum gguf_type type) {@@ -18988,24 +18808,29 @@ int gguf_find_key(const struct gguf_context * ctx, const char * key) {}const char * gguf_get_key(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));return ctx->kv[key_id].key.data;}enum gguf_type gguf_get_kv_type(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));return ctx->kv[key_id].type;}enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);return ctx->kv[key_id].value.arr.type;}const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);return ctx->kv[key_id].value.arr.data;}const char * gguf_get_arr_str(const struct gguf_context * ctx, int key_id, int i) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);struct gguf_kv * kv = &ctx->kv[key_id];struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[i];@@ -19013,70 +18838,90 @@ const char * gguf_get_arr_str(const struct gguf_context * ctx, int key_id, int i}int gguf_get_arr_n(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_ARRAY);return ctx->kv[key_id].value.arr.n;}uint8_t gguf_get_val_u8(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT8);return ctx->kv[key_id].value.uint8;}int8_t gguf_get_val_i8(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT8);return ctx->kv[key_id].value.int8;}uint16_t gguf_get_val_u16(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT16);return ctx->kv[key_id].value.uint16;}int16_t gguf_get_val_i16(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT16);return ctx->kv[key_id].value.int16;}uint32_t gguf_get_val_u32(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT32);return ctx->kv[key_id].value.uint32;}int32_t gguf_get_val_i32(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT32);return ctx->kv[key_id].value.int32;}float gguf_get_val_f32(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_FLOAT32);return ctx->kv[key_id].value.float32;}uint64_t gguf_get_val_u64(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_UINT64);return ctx->kv[key_id].value.uint64;}int64_t gguf_get_val_i64(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_INT64);return ctx->kv[key_id].value.int64;}double gguf_get_val_f64(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_FLOAT64);return ctx->kv[key_id].value.float64;}bool gguf_get_val_bool(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_BOOL);return ctx->kv[key_id].value.bool_;}const char * gguf_get_val_str(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));GGML_ASSERT(ctx->kv[key_id].type == GGUF_TYPE_STRING);return ctx->kv[key_id].value.str.data;}+const void * gguf_get_val_data(const struct gguf_context * ctx, int key_id) {+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));+ GGML_ASSERT(ctx->kv[key_id].type != GGUF_TYPE_ARRAY);+ GGML_ASSERT(ctx->kv[key_id].type != GGUF_TYPE_STRING);+ return &ctx->kv[key_id].value;+}+int gguf_get_n_tensors(const struct gguf_context * ctx) {return ctx->header.n_tensors;}@@ -19607,11 +19452,7 @@ int ggml_cpu_has_wasm_simd(void) {}int ggml_cpu_has_blas(void) {-#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_TINYBLAS)- return 1;-#else- return 0;-#endif+ return ggml_cpu_has_cublas();}int ggml_cpu_has_cublas(void) {diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex db102bc8..1cff846a 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -1,4 +1,4 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once@@ -60,7 +60,8 @@// {// ...//-// struct ggml_cgraph gf = ggml_build_forward(f);+// struct ggml_cgraph * gf = ggml_new_graph(ctx);+// ggml_build_forward_expand(gf, f);//// // set the input variable and parameter values// ggml_set_f32(x, 2.0f);@@ -217,15 +218,14 @@#define GGML_QNT_VERSION 2 // bump this on quantization format changes#define GGML_QNT_VERSION_FACTOR 1000 // do not change this-#define GGML_MAX_DIMS 4-#define GGML_MAX_NODES 16384-#define GGML_MAX_PARAMS 1024-#define GGML_MAX_CONTEXTS 64-#define GGML_MAX_SRC 6-#define GGML_MAX_NAME 64-#define GGML_MAX_OP_PARAMS 64-#define GGML_DEFAULT_N_THREADS 4-+#define GGML_MAX_DIMS 4+#define GGML_MAX_PARAMS 1024+#define GGML_MAX_CONTEXTS 64+#define GGML_MAX_SRC 10+#define GGML_MAX_NAME 64+#define GGML_MAX_OP_PARAMS 64+#define GGML_DEFAULT_N_THREADS 4+#define GGML_DEFAULT_GRAPH_SIZE 2048#if UINTPTR_MAX == 0xFFFFFFFF#define GGML_MEM_ALIGN 4#else@@ -248,6 +248,7 @@#define GGML_ASSERT(x) \do { \if (!(x)) { \+ fflush(stdout); \fprintf(stderr, "GGML_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \abort(); \} \@@ -285,6 +286,20 @@const type prefix##3 = (pointer)->array[3]; \GGML_UNUSED(prefix##3);+#define GGML_TENSOR_UNARY_OP_LOCALS \+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)++#define GGML_TENSOR_BINARY_OP_LOCALS \+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)+#ifdef __cplusplusextern "C" {#endif@@ -383,6 +398,7 @@ extern "C" {GGML_OP_GROUP_NORM,GGML_OP_MUL_MAT,+ GGML_OP_MUL_MAT_ID,GGML_OP_OUT_PROD,GGML_OP_SCALE,@@ -404,18 +420,13 @@ extern "C" {GGML_OP_ROPE_BACK,GGML_OP_ALIBI,GGML_OP_CLAMP,- GGML_OP_CONV_1D,- GGML_OP_CONV_1D_STAGE_0, // internal- GGML_OP_CONV_1D_STAGE_1, // internalGGML_OP_CONV_TRANSPOSE_1D,- GGML_OP_CONV_2D,- GGML_OP_CONV_2D_STAGE_0, // internal- GGML_OP_CONV_2D_STAGE_1, // internal+ GGML_OP_IM2COL,GGML_OP_CONV_TRANSPOSE_2D,GGML_OP_POOL_1D,GGML_OP_POOL_2D,-GGML_OP_UPSCALE, // nearest interpolate+ GGML_OP_ARGSORT,GGML_OP_FLASH_ATTN,GGML_OP_FLASH_FF,@@ -455,6 +466,9 @@ extern "C" {GGML_UNARY_OP_GELU,GGML_UNARY_OP_GELU_QUICK,GGML_UNARY_OP_SILU,+ GGML_UNARY_OP_LEAKY,++ GGML_UNARY_OP_COUNT,};enum ggml_object_type {@@ -535,37 +549,33 @@ extern "C" {int n_threads;- // the `n_tasks` of nodes, 1:1 mapping to cgraph nodes- int n_tasks[GGML_MAX_NODES];-// abort ggml_graph_compute when truebool (*abort_callback)(void * data);void * abort_callback_data;};- // next prime after GGML_MAX_NODES- // #define GGML_GRAPH_HASHTABLE_SIZE 4099- // next prime after GGML_MAX_NODES * 2 (nodes + leafs)- // #define GGML_GRAPH_HASHTABLE_SIZE 8273- // #define GGML_GRAPH_HASHTABLE_SIZE 16411- #define GGML_GRAPH_HASHTABLE_SIZE 32771-enum ggml_cgraph_eval_order {GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,GGML_CGRAPH_EVAL_ORDER_COUNT};+ struct ggml_hash_set {+ size_t size;+ struct ggml_tensor ** keys;+ };+// computation graphstruct ggml_cgraph {+ int size;int n_nodes;int n_leafs;- struct ggml_tensor * nodes[GGML_MAX_NODES];- struct ggml_tensor * grads[GGML_MAX_NODES];- struct ggml_tensor * leafs[GGML_MAX_NODES];+ struct ggml_tensor ** nodes;+ struct ggml_tensor ** grads;+ struct ggml_tensor ** leafs;- void * visited_hash_table[GGML_GRAPH_HASHTABLE_SIZE];+ struct ggml_hash_set visited_hash_table;enum ggml_cgraph_eval_order order;@@ -575,8 +585,6 @@ extern "C" {int64_t perf_time_us;};- static const size_t GGML_GRAPH_SIZE = sizeof(struct ggml_cgraph);-// scratch bufferstruct ggml_scratch {size_t offs;@@ -621,6 +629,8 @@ extern "C" {GGML_API int64_t ggml_cycles(void);GGML_API int64_t ggml_cycles_per_ms(void);+ GGML_API void ggml_print_backtrace(void);+GGML_API void ggml_numa_init(void); // call once for better performance on NUMA systemsGGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node@@ -641,6 +651,9 @@ extern "C" {GGML_API const char * ggml_op_name (enum ggml_op op);GGML_API const char * ggml_op_symbol(enum ggml_op op);+ GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);+ GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name+GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);GGML_API bool ggml_is_quantized(enum ggml_type type);@@ -713,7 +726,7 @@ extern "C" {// Context tensor enumeration and lookupGGML_API struct ggml_tensor * ggml_get_first_tensor(struct ggml_context * ctx);GGML_API struct ggml_tensor * ggml_get_next_tensor (struct ggml_context * ctx, struct ggml_tensor * tensor);- GGML_API struct ggml_tensor * ggml_get_tensor (struct ggml_context * ctx, const char * name);+ GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);@@ -947,6 +960,10 @@ extern "C" {struct ggml_context * ctx,struct ggml_tensor * a);+ GGML_API struct ggml_tensor * ggml_leaky(+ struct ggml_context * ctx,+ struct ggml_tensor * a);+GGML_API struct ggml_tensor * ggml_relu_inplace(struct ggml_context * ctx,struct ggml_tensor * a);@@ -1033,6 +1050,16 @@ extern "C" {struct ggml_tensor * a,struct ggml_tensor * b);+ // indirect matrix multiplication+ // ggml_mul_mat_id(ctx, as, ids, id, b) ~= ggml_mul_mat(as[ids[id]], b)+ GGML_API struct ggml_tensor * ggml_mul_mat_id(+ struct ggml_context * ctx,+ struct ggml_tensor * const as[],+ int n_as,+ struct ggml_tensor * ids,+ int id,+ struct ggml_tensor * b);+// A: m columns, n rows,// B: p columns, n rows,// result is m columns, p rows@@ -1240,6 +1267,7 @@ extern "C" {struct ggml_context * ctx,struct ggml_tensor * a);+ // supports 3D: a->ne[2] == b->ne[1]GGML_API struct ggml_tensor * ggml_get_rows(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1288,6 +1316,14 @@ extern "C" {struct ggml_context * ctx,struct ggml_tensor * a);+ // fused soft_max(a*scale + mask)+ // mask is optional+ GGML_API struct ggml_tensor * ggml_soft_max_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * mask,+ float scale);+GGML_API struct ggml_tensor * ggml_soft_max_back(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1403,6 +1439,18 @@ extern "C" {float min,float max);+ GGML_API struct ggml_tensor * ggml_im2col(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int s0,+ int s1,+ int p0,+ int p1,+ int d0,+ int d1,+ bool is_2D);+GGML_API struct ggml_tensor * ggml_conv_1d(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1486,6 +1534,8 @@ extern "C" {int s0, // strideint p0); // padding+ // the result will have 2*p0 padding for the first dimension+ // and 2*p1 padding for the second dimensionGGML_API struct ggml_tensor * ggml_pool_2d(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1494,8 +1544,8 @@ extern "C" {int k1,int s0,int s1,- int p0,- int p1);+ float p0,+ float p1);// nearest interpolate// used in stable-diffusion@@ -1504,6 +1554,23 @@ extern "C" {struct ggml_tensor * a,int scale_factor);+ // sort rows+ enum ggml_sort_order {+ GGML_SORT_ASC,+ GGML_SORT_DESC,+ };++ GGML_API struct ggml_tensor * ggml_argsort(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_sort_order order);++ // top k elements per row+ GGML_API struct ggml_tensor * ggml_top_k(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int k);+GGML_API struct ggml_tensor * ggml_flash_attn(struct ggml_context * ctx,struct ggml_tensor * q,@@ -1565,7 +1632,6 @@ extern "C" {int kh);// used in sam-GGML_API struct ggml_tensor * ggml_add_rel_pos(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1736,19 +1802,22 @@ extern "C" {GGML_API void ggml_build_forward_expand (struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);GGML_API void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, bool keep);- GGML_API struct ggml_cgraph ggml_build_forward (struct ggml_tensor * tensor);- GGML_API struct ggml_cgraph ggml_build_backward(struct ggml_context * ctx, struct ggml_cgraph * gf, bool keep);-// graph allocation in a context- GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx);- GGML_API struct ggml_cgraph * ggml_build_forward_ctx(struct ggml_context * ctx, struct ggml_tensor * tensor);+ GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false+ GGML_API struct ggml_cgraph * ggml_new_graph_custom (struct ggml_context * ctx, size_t size, bool grads);+ GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph);+ GGML_API struct ggml_cgraph ggml_graph_view (struct ggml_cgraph * cgraph, int i0, int i1);+ GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);+ GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // zero grads+ GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);+GGML_API size_t ggml_graph_overhead(void);+ GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);// ggml_graph_plan() has to be called before ggml_graph_compute()// when plan.work_size > 0, caller must allocate memory for plan.work_dataGGML_API struct ggml_cplan ggml_graph_plan (struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);- GGML_API int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);- GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph);+ GGML_API int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);// same as ggml_graph_compute() but the work data is allocated as a part of the context// note: the drawback of this API is that you must have ensured that the context has enough memory for the work data@@ -1756,8 +1825,8 @@ extern "C" {GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name);- GGML_API void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname);- GGML_API struct ggml_cgraph ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval);+ GGML_API void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname);+ GGML_API struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval);// print info and performance information for the graphGGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);@@ -1820,6 +1889,8 @@ extern "C" {struct ggml_opt_params {enum ggml_opt_type type;+ size_t graph_size;+int n_threads;// delta-based convergence test@@ -2031,6 +2102,7 @@ extern "C" {GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int key_id);GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int key_id);GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int key_id);+ GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int key_id);GGML_API int gguf_get_arr_n (const struct gguf_context * ctx, int key_id);GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);diff --git a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.incindex 19dbd698..25851025 100644--- a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc@@ -13,7 +13,7 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *const int32x4_t vzero = vdupq_n_s32(0);#endif- int8x16x2_t q2bytes;+ ggml_int8x16x2_t q2bytes;uint8_t aux[16];float sum = 0;@@ -32,8 +32,8 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *vst1q_u8(aux, scales);const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);- const int16x8x2_t q8sums = vld1q_s16_x2(y[i].bsums);- const int16x8x2_t mins16 = {vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))};+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);+ const ggml_int16x8x2_t mins16 = {vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))};const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),@@ -61,7 +61,7 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *#endif#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\- q8bytes = vld1q_s8_x2(q8); q8 += 32;\+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\MULTIPLY_ACCUM_WITH_SCALE((index));@@ -69,9 +69,9 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *for (int j = 0; j < QK_K/128; ++j) {- const uint8x16x2_t q2bits = vld1q_u8_x2(q2); q2 += 32;+ const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;- int8x16x2_t q8bytes = vld1q_s8_x2(q8); q8 += 32;+ ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));MULTIPLY_ACCUM_WITH_SCALE(0);diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.incindex 66929626..389e3738 100644--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc@@ -25,7 +25,7 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void *const uint8x16_t m3 = vshlq_n_u8(m0, 3);const int8_t m32 = 32;- int8x16x4_t q3bytes;+ ggml_int8x16x4_t q3bytes;float sum = 0;@@ -37,9 +37,9 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void *const uint8_t * restrict qh = x[i].hmask;const int8_t * restrict q8 = y[i].qs;- uint8x16x2_t qhbits = vld1q_u8_x2(qh);+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);- uint8x16x4_t q3h;+ ggml_uint8x16x4_t q3h;int32_t isum = 0;@@ -55,9 +55,9 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void *for (int j = 0; j < QK_K/128; ++j) {- const uint8x16x2_t q3bits = vld1q_u8_x2(q3); q3 += 32;- const int8x16x4_t q8bytes_1 = vld1q_s8_x4(q8); q8 += 64;- const int8x16x4_t q8bytes_2 = vld1q_s8_x4(q8); q8 += 64;+ const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;+ const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;+ const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);diff --git a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q4_1_q8_1.incnew file mode 100644index 00000000..12ec2787--- /dev/null+++ b/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc@@ -0,0 +1,175 @@+static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ const int qk = QK8_1;+ const int nb = n / qk;++ assert(n % qk == 0);++ const block_q4_1 * restrict x = vx;+ const block_q8_1 * restrict y = vy;++ // TODO: add WASM SIMD+#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ float summs = 0;++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q4_1 * restrict x0 = &x[i + 0];+ const block_q4_1 * restrict x1 = &x[i + 1];+ const block_q8_1 * restrict y0 = &y[i + 0];+ const block_q8_1 * restrict y1 = &y[i + 1];++ summs += GGML_FP16_TO_FP32(x0->m) * y0->s + GGML_FP16_TO_FP32(x1->m) * y1->s;++ const uint8x16_t m4b = vdupq_n_u8(0x0F);++ const uint8x16_t v0_0 = vld1q_u8(x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);++ // 4-bit -> 8-bit+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // load y+ const int8x16_t v1_0l = vld1q_s8(y0->qs);+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);+ const int8x16_t v1_1l = vld1q_s8(y1->qs);+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);++#if defined(__ARM_FEATURE_DOTPROD)+ // dot product into int32x4_t+ const int32x4_t p_0 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);+ const int32x4_t p_1 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*y0->d);+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*y1->d);+#else+ const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0l), vget_low_s8 (v1_0l));+ const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0l), vget_high_s8(v1_0l));+ const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0h), vget_low_s8 (v1_0h));+ const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0h), vget_high_s8(v1_0h));++ const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1l), vget_low_s8 (v1_1l));+ const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1l), vget_high_s8(v1_1l));+ const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1h), vget_low_s8 (v1_1h));+ const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1h), vget_high_s8(v1_1h));++ const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));+ const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));+ const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));+ const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*y0->d);+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*y1->d);+#endif+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;+#elif defined(__AVX2__) || defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ float summs = 0;++ // Main loop+ for (int i = 0; i < nb; ++i) {+#ifdef __AVX2__+ const float d0 = GGML_FP16_TO_FP32__F16C(x[i].d);+#else+ const float d0 = GGML_FP16_TO_FP32(x[i].d);+#endif+ const float d1 = y[i].d;++#ifdef __AVX2__+ summs += GGML_FP16_TO_FP32__F16C(x[i].m) * y[i].s;+#else+ summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;+#endif++ const __m256 d0v = _mm256_set1_ps( d0 );+ const __m256 d1v = _mm256_set1_ps( d1 );++ // Compute combined scales+ const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );++ // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes+#if defined(__AVX2__)+ const __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);+#else+ const __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);+#endif+ const __m256i by = _mm256_loadu_si256( (const __m256i *)y[i].qs );++#if defined(__AVX2__)+ const __m256 xy = mul_sum_us8_pairs_float_avx2(bx, by);+#else+ const __m256 xy = mul_sum_us8_pairs_float_avx(bx, by);+#endif++ // Accumulate d0*d1*x*y+#if defined(__AVX2__)+ acc = _mm256_fmadd_ps( d0d1, xy, acc );+#else+ acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );+#endif+ }++ *s = hsum_float_8(acc) + summs;+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;++ size_t vl = __riscv_vsetvl_e8m1(qk/2);++ for (int i = 0; i < nb; i++) {+ // load elements+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);++ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);++ // mask and store lower part of x, and then upper part+ vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);+ vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);++ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);++ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);++ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);++ sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ int sumi = 0;++ for (int j = 0; j < qk/2; ++j) {+ const int v0 = (x[i].qs[j] & 0x0F);+ const int v1 = (x[i].qs[j] >> 4);++ sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);+ }++ sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;+ }++ *s = sumf;+#endif+}diff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.incindex 0157f32b..cfa128fa 100644--- a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc@@ -19,8 +19,8 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void *const int32x4_t mzero = vdupq_n_s32(0);#endif- int8x16x2_t q4bytes;- int8x16x2_t q8bytes;+ ggml_int8x16x2_t q4bytes;+ ggml_int8x16x2_t q8bytes;float sumf = 0;@@ -55,17 +55,17 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void *for (int j = 0; j < QK_K/64; ++j) {- const uint8x16x2_t q4bits = vld1q_u8_x2(q4); q4 += 32;+ const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;#ifdef __ARM_FEATURE_DOTPROD- q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));const int32x4_t p1 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);sumi1 += vaddvq_s32(p1) * scales[2*j+0];- q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));@@ -73,7 +73,7 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void *sumi2 += vaddvq_s32(p2) * scales[2*j+1];#else- q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),@@ -82,7 +82,7 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void *vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));sumi1 += vaddvq_s16(vaddq_s16(p0, p1)) * scales[2*j+0];- q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),@@ -109,8 +109,8 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void *for (int i = 0; i < nb; ++i) {- const float d = y[i].d * _cvtsh_ss(x[i].d);- const float dmin = -y[i].d * _cvtsh_ss(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);memcpy(utmp, x[i].scales, 12);utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);diff --git a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q5_0_q8_0.incnew file mode 100644index 00000000..430a9f16--- /dev/null+++ b/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc@@ -0,0 +1,305 @@+static void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(qk == QK5_0);++ const block_q5_0 * restrict x = vx;+ const block_q8_0 * restrict y = vy;++#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ uint32_t qh0;+ uint32_t qh1;++ uint64_t tmp0[4];+ uint64_t tmp1[4];++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q5_0 * restrict x0 = &x[i];+ const block_q5_0 * restrict x1 = &x[i + 1];+ const block_q8_0 * restrict y0 = &y[i];+ const block_q8_0 * restrict y1 = &y[i + 1];++ const uint8x16_t m4b = vdupq_n_u8(0x0F);++ // extract the 5th bit via lookup table ((!b) << 4)+ memcpy(&qh0, x0->qh, sizeof(qh0));+ memcpy(&qh1, x1->qh, sizeof(qh1));++ tmp0[0] = table_b2b_1[(qh0 >> 0) & 0xFF];+ tmp0[1] = table_b2b_1[(qh0 >> 8) & 0xFF];+ tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];+ tmp0[3] = table_b2b_1[(qh0 >> 24) ];++ tmp1[0] = table_b2b_1[(qh1 >> 0) & 0xFF];+ tmp1[1] = table_b2b_1[(qh1 >> 8) & 0xFF];+ tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];+ tmp1[3] = table_b2b_1[(qh1 >> 24) ];++ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));++ const uint8x16_t v0_0 = vld1q_u8(x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);++ // 4-bit -> 8-bit+ int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)+ const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);+ const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);+ const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);+ const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);++ // load y+ const int8x16_t v1_0l = vld1q_s8(y0->qs);+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);+ const int8x16_t v1_1l = vld1q_s8(y1->qs);+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);++#if defined(__ARM_FEATURE_DOTPROD)+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(+ vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),+ vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(+ vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),+ vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#else+ const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0lf), vget_low_s8 (v1_0l));+ const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0lf), vget_high_s8(v1_0l));+ const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hf), vget_low_s8 (v1_0h));+ const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hf), vget_high_s8(v1_0h));++ const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1lf), vget_low_s8 (v1_1l));+ const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1lf), vget_high_s8(v1_1l));+ const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hf), vget_low_s8 (v1_1h));+ const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hf), vget_high_s8(v1_1h));++ const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));+ const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));+ const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));+ const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#endif+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);+#elif defined(__wasm_simd128__)+ v128_t sumv = wasm_f32x4_splat(0.0f);++ uint32_t qh;+ uint64_t tmp[4];++ // TODO: check if unrolling this is better+ for (int i = 0; i < nb; ++i) {+ const block_q5_0 * restrict x0 = &x[i];+ const block_q8_0 * restrict y0 = &y[i];++ const v128_t m4b = wasm_i8x16_splat(0x0F);++ // extract the 5th bit+ memcpy(&qh, x0->qh, sizeof(qh));++ tmp[0] = table_b2b_1[(qh >> 0) & 0xFF];+ tmp[1] = table_b2b_1[(qh >> 8) & 0xFF];+ tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];+ tmp[3] = table_b2b_1[(qh >> 24) ];++ const v128_t qhl = wasm_v128_load(tmp + 0);+ const v128_t qhh = wasm_v128_load(tmp + 2);++ const v128_t v0 = wasm_v128_load(x0->qs);++ // 4-bit -> 8-bit+ const v128_t v0l = wasm_v128_and (v0, m4b);+ const v128_t v0h = wasm_u8x16_shr(v0, 4);++ // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)+ const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);+ const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);++ // load y+ const v128_t v1l = wasm_v128_load(y0->qs);+ const v128_t v1h = wasm_v128_load(y0->qs + 16);++ // int8x16 -> int16x8+ const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);+ const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);+ const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);+ const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);++ const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);+ const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);+ const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);+ const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);++ // dot product+ sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(+ wasm_i32x4_add(+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),+ wasm_i32x4_dot_i16x8(v0lfh, v1lh)),+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),+ wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),+ wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));+ }++ *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) ++ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);+#elif defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (int i = 0; i < nb; i++) {+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d));++ __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);+ __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);+ bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));+ bx = _mm256_or_si256(bx, bxhi);++ __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);++ /* Multiply q with scale and accumulate */+ acc = _mm256_fmadd_ps(d, q, acc);+ }++ *s = hsum_float_8(acc);+#elif defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();+ __m128i mask = _mm_set1_epi8((char)0xF0);++ // Main loop+ for (int i = 0; i < nb; i++) {+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));++ __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);+ const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);+ __m128i bxhil = _mm256_castsi256_si128(bxhi);+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);+ bxhil = _mm_andnot_si128(bxhil, mask);+ bxhih = _mm_andnot_si128(bxhih, mask);+ __m128i bxl = _mm256_castsi256_si128(bx);+ __m128i bxh = _mm256_extractf128_si256(bx, 1);+ bxl = _mm_or_si128(bxl, bxhil);+ bxh = _mm_or_si128(bxh, bxhih);+ bx = MM256_SET_M128I(bxh, bxl);++ const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_i8_pairs_float_avx(bx, by);++ /* Multiply q with scale and accumulate */+ acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);+ }++ *s = hsum_float_8(acc);+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;++ uint32_t qh;++ size_t vl = __riscv_vsetvl_e8m1(qk/2);++ // These tempory registers are for masking and shift operations+ vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);+ vuint32m2_t vt_2 = __riscv_vsll_vv_u32m2(__riscv_vmv_v_x_u32m2(1, vl), vt_1, vl);++ vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);+ vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);++ for (int i = 0; i < nb; i++) {+ memcpy(&qh, x[i].qh, sizeof(uint32_t));++ // ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(xha_0, vt_1, vl);+ vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);++ // ((qh & (1u << (j + 16))) >> (j + 12));+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(vt_3, qh, vl);+ vuint32m2_t xhl_1 = __riscv_vsrl_vv_u32m2(xha_1, vt_4, vl);++ // narrowing+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xhl_0, vl);+ vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);++ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xhl_1, vl);+ vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);++ // load+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);++ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);++ vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);+ vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);++ vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);+ vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);++ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 16, vl);+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 16, vl);++ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);++ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);++ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);++ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ uint32_t qh;+ memcpy(&qh, x[i].qh, sizeof(qh));++ int sumi = 0;++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;+ const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));++ const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;+ const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;++ sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);+ }++ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;+ }++ *s = sumf;+#endif+}diff --git a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q5_1_q8_1.incnew file mode 100644index 00000000..8d6932fc--- /dev/null+++ b/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc@@ -0,0 +1,318 @@+static void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ const int qk = QK8_1;+ const int nb = n / qk;++ assert(n % qk == 0);+ assert(qk == QK5_1);++ const block_q5_1 * restrict x = vx;+ const block_q8_1 * restrict y = vy;++#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ float summs0 = 0.0f;+ float summs1 = 0.0f;++ uint32_t qh0;+ uint32_t qh1;++ uint64_t tmp0[4];+ uint64_t tmp1[4];++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q5_1 * restrict x0 = &x[i];+ const block_q5_1 * restrict x1 = &x[i + 1];+ const block_q8_1 * restrict y0 = &y[i];+ const block_q8_1 * restrict y1 = &y[i + 1];++ const uint8x16_t m4b = vdupq_n_u8(0x0F);++ summs0 += GGML_FP16_TO_FP32(x0->m) * y0->s;+ summs1 += GGML_FP16_TO_FP32(x1->m) * y1->s;++ // extract the 5th bit via lookup table ((b) << 4)+ memcpy(&qh0, x0->qh, sizeof(qh0));+ memcpy(&qh1, x1->qh, sizeof(qh1));++ tmp0[0] = table_b2b_0[(qh0 >> 0) & 0xFF];+ tmp0[1] = table_b2b_0[(qh0 >> 8) & 0xFF];+ tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];+ tmp0[3] = table_b2b_0[(qh0 >> 24) ];++ tmp1[0] = table_b2b_0[(qh1 >> 0) & 0xFF];+ tmp1[1] = table_b2b_0[(qh1 >> 8) & 0xFF];+ tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];+ tmp1[3] = table_b2b_0[(qh1 >> 24) ];++ const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));+ const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));+ const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));+ const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));++ const uint8x16_t v0_0 = vld1q_u8(x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);++ // 4-bit -> 8-bit+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // add high bit+ const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);+ const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);+ const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);+ const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);++ // load y+ const int8x16_t v1_0l = vld1q_s8(y0->qs);+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);+ const int8x16_t v1_1l = vld1q_s8(y1->qs);+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);++#if defined(__ARM_FEATURE_DOTPROD)+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(+ vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),+ vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*y0->d);+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(+ vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),+ vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*y1->d);+#else+ const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0lf), vget_low_s8 (v1_0l));+ const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0lf), vget_high_s8(v1_0l));+ const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hf), vget_low_s8 (v1_0h));+ const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hf), vget_high_s8(v1_0h));++ const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1lf), vget_low_s8 (v1_1l));+ const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1lf), vget_high_s8(v1_1l));+ const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hf), vget_low_s8 (v1_1h));+ const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hf), vget_high_s8(v1_1h));++ const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));+ const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));+ const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));+ const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*y0->d);+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*y1->d);+#endif+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;+#elif defined(__wasm_simd128__)+ v128_t sumv = wasm_f32x4_splat(0.0f);++ float summs = 0.0f;++ uint32_t qh;+ uint64_t tmp[4];++ // TODO: check if unrolling this is better+ for (int i = 0; i < nb; ++i) {+ const block_q5_1 * restrict x0 = &x[i];+ const block_q8_1 * restrict y0 = &y[i];++ summs += GGML_FP16_TO_FP32(x0->m) * y0->s;++ const v128_t m4b = wasm_i8x16_splat(0x0F);++ // extract the 5th bit+ memcpy(&qh, x0->qh, sizeof(qh));++ tmp[0] = table_b2b_0[(qh >> 0) & 0xFF];+ tmp[1] = table_b2b_0[(qh >> 8) & 0xFF];+ tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];+ tmp[3] = table_b2b_0[(qh >> 24) ];++ const v128_t qhl = wasm_v128_load(tmp + 0);+ const v128_t qhh = wasm_v128_load(tmp + 2);++ const v128_t v0 = wasm_v128_load(x0->qs);++ // 4-bit -> 8-bit+ const v128_t v0l = wasm_v128_and (v0, m4b);+ const v128_t v0h = wasm_u8x16_shr(v0, 4);++ // add high bit+ const v128_t v0lf = wasm_v128_or(v0l, qhl);+ const v128_t v0hf = wasm_v128_or(v0h, qhh);++ // load y+ const v128_t v1l = wasm_v128_load(y0->qs);+ const v128_t v1h = wasm_v128_load(y0->qs + 16);++ // int8x16 -> int16x8+ const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);+ const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);+ const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);+ const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);++ const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);+ const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);+ const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);+ const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);++ // dot product+ sumv = wasm_f32x4_add(sumv,+ wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),+ wasm_i32x4_dot_i16x8(v0lfh, v1lh)),+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),+ wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),+ wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * y0->d)));+ }++ *s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) ++ wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;+#elif defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ float summs = 0.0f;++ // Main loop+ for (int i = 0; i < nb; i++) {+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d));++ summs += GGML_FP16_TO_FP32__F16C(x[i].m) * y[i].s;++ __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);+ __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);+ bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));+ bx = _mm256_or_si256(bx, bxhi);++ const __m256 dy = _mm256_set1_ps(y[i].d);+ const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_us8_pairs_float_avx2(bx, by);++ acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);+ }++ *s = hsum_float_8(acc) + summs;+#elif defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();+ __m128i mask = _mm_set1_epi8(0x10);++ float summs = 0.0f;++ // Main loop+ for (int i = 0; i < nb; i++) {+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));++ summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;++ __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);+ const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);+ __m128i bxhil = _mm256_castsi256_si128(bxhi);+ __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);+ bxhil = _mm_and_si128(bxhil, mask);+ bxhih = _mm_and_si128(bxhih, mask);+ __m128i bxl = _mm256_castsi256_si128(bx);+ __m128i bxh = _mm256_extractf128_si256(bx, 1);+ bxl = _mm_or_si128(bxl, bxhil);+ bxh = _mm_or_si128(bxh, bxhih);+ bx = MM256_SET_M128I(bxh, bxl);++ const __m256 dy = _mm256_set1_ps(y[i].d);+ const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_us8_pairs_float_avx(bx, by);++ acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);+ }++ *s = hsum_float_8(acc) + summs;+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;++ uint32_t qh;++ size_t vl = __riscv_vsetvl_e8m1(qk/2);++ // temporary registers for shift operations+ vuint32m2_t vt_1 = __riscv_vid_v_u32m2(vl);+ vuint32m2_t vt_2 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);++ for (int i = 0; i < nb; i++) {+ memcpy(&qh, x[i].qh, sizeof(uint32_t));++ // load qh+ vuint32m2_t vqh = __riscv_vmv_v_x_u32m2(qh, vl);++ // ((qh >> (j + 0)) << 4) & 0x10;+ vuint32m2_t xhr_0 = __riscv_vsrl_vv_u32m2(vqh, vt_1, vl);+ vuint32m2_t xhl_0 = __riscv_vsll_vx_u32m2(xhr_0, 4, vl);+ vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(xhl_0, 0x10, vl);++ // ((qh >> (j + 12)) ) & 0x10;+ vuint32m2_t xhr_1 = __riscv_vsrl_vv_u32m2(vqh, vt_2, vl);+ vuint32m2_t xha_1 = __riscv_vand_vx_u32m2(xhr_1, 0x10, vl);++ // narrowing+ vuint16m1_t xhc_0 = __riscv_vncvt_x_x_w_u16m1(xha_0, vl);+ vuint8mf2_t xh_0 = __riscv_vncvt_x_x_w_u8mf2(xhc_0, vl);++ vuint16m1_t xhc_1 = __riscv_vncvt_x_x_w_u16m1(xha_1, vl);+ vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);++ // load+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);++ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);++ vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);+ vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);++ vuint8mf2_t x_a = __riscv_vor_vv_u8mf2(x_at, xh_0, vl);+ vuint8mf2_t x_l = __riscv_vor_vv_u8mf2(x_lt, xh_1, vl);++ vint8mf2_t v0 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t v1 = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);++ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);++ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);++ sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ uint32_t qh;+ memcpy(&qh, x[i].qh, sizeof(qh));++ int sumi = 0;++ for (int j = 0; j < qk/2; ++j) {+ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;+ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;++ const int32_t x0 = (x[i].qs[j] & 0xF) | xh_0;+ const int32_t x1 = (x[i].qs[j] >> 4) | xh_1;++ sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);+ }++ sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;+ }++ *s = sumf;+#endif+}diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.incindex 4bbc5061..e1fae962 100644--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc@@ -22,7 +22,7 @@ static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void *const int32x4_t mzero = vdupq_n_s32(0);#endif- int8x16x4_t q5bytes;+ ggml_int8x16x4_t q5bytes;float sumf = 0;@@ -52,16 +52,16 @@ static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void *const uint8_t * restrict qh = x[i].qh;const int8_t * restrict q8 = y[i].qs;- uint8x16x2_t qhbits = vld1q_u8_x2(qh);+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);- uint8x16x4_t q5h;+ ggml_uint8x16x4_t q5h;int32_t sumi = 0;for (int j = 0; j < QK_K/64; ++j) {- const uint8x16x2_t q5bits = vld1q_u8_x2(q5); q5 += 32;- const int8x16x4_t q8bytes = vld1q_s8_x4(q8); q8 += 64;+ const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;+ const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);diff --git a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.incindex 8a32dc31..775a6be0 100644--- a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc@@ -18,8 +18,8 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *const uint8x16_t mone = vdupq_n_u8(3);- int8x16x4_t q6bytes;- uint8x16x4_t q6h;+ ggml_int8x16x4_t q6bytes;+ ggml_uint8x16x4_t q6h;for (int i = 0; i < nb; ++i) {@@ -31,9 +31,9 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *const int8_t * restrict scale = x[i].scales;- const int16x8x2_t q8sums = vld1q_s16_x2(y[i].bsums);+ const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);const int8x16_t scales = vld1q_s8(scale);- const int16x8x2_t q6scales = {vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))};+ const ggml_int16x8x2_t q6scales = {vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))};const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),@@ -45,9 +45,9 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *for (int j = 0; j < QK_K/128; ++j) {- uint8x16x2_t qhbits = vld1q_u8_x2(qh); qh += 32;- uint8x16x4_t q6bits = vld1q_u8_x4(q6); q6 += 64;- int8x16x4_t q8bytes = vld1q_s8_x4(q8); q8 += 64;+ ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;+ ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;+ ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);@@ -90,7 +90,7 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *scale += 2;#endif- q8bytes = vld1q_s8_x4(q8); q8 += 64;+ q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;shifted = vshrq_n_u8(qhbits.val[0], 4);q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);@@ -155,7 +155,7 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);const uint8_t * restrict q4 = x[i].ql;const uint8_t * restrict qh = x[i].qh;diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.incindex d5c067dc..e6dd15b8 100644--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc@@ -78,11 +78,14 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void *#if defined(__AVX2__)const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);- // Multiply q with scale and accumulate- acc = _mm256_fmadd_ps( d, q, acc );#elseconst __m256 q = mul_sum_i8_pairs_float_avx(bx, by);+#endif+// Multiply q with scale and accumulate+#if defined(__AVX2__)+ acc = _mm256_fmadd_ps( d, q, acc );+#elseacc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );#endif}diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 6467b187..63bf2729 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -8,14 +8,20 @@#include "runtime.h"#include "ggml.h"+#include "ggml-alloc.h"#include "ggml-metal.h"#include "ggml-cuda.h"-#if defined(GGML_USE_CLBLAST)+#ifdef GGML_USE_CUBLAS+# include "ggml-cuda.h"+#elif defined(GGML_USE_CLBLAST)# include "ggml-opencl.h"#endif+#ifdef GGML_USE_METAL+# include "ggml-metal.h"+#endif#ifdef GGML_USE_MPI# include "ggml-mpi.h"#endif@@ -28,15 +34,24 @@#endif#ifdef __has_include-#if __has_include(<unistd.h>)-#include <unistd.h>-#if defined(_POSIX_MAPPED_FILES)-#include <sys/mman.h>-#endif-#if defined(_POSIX_MEMLOCK_RANGE)-#include <sys/resource.h>-#endif+ #if __has_include(<unistd.h>)+ #include <unistd.h>+ #if defined(_POSIX_MAPPED_FILES)+ #include <sys/mman.h>+ #endif+ #if defined(_POSIX_MEMLOCK_RANGE)+ #include <sys/resource.h>+ #endif+ #endif#endif++#if defined(_WIN32)+ #define WIN32_LEAN_AND_MEAN+ #ifndef NOMINMAX+ #define NOMINMAX+ #endif+ #include <windows.h>+ #include <io.h>#endif#include <algorithm>@@ -67,8 +82,13 @@#include <set>#include <sstream>#include <thread>+#include <type_traits>#include <unordered_map>+#if defined(_MSC_VER)+#pragma warning(disable: 4244 4267) // possible loss of data+#endif+#ifdef __GNUC__#ifdef __MINGW32__#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))@@ -79,6 +99,9 @@#define LLAMA_ATTRIBUTE_FORMAT(...)#endif+#define LLAMA_MAX_NODES 8192+#define LLAMA_MAX_EXPERTS 8+//// logging//@@ -192,6 +215,7 @@ enum llm_arch {LLM_ARCH_REFACT,LLM_ARCH_BLOOM,LLM_ARCH_STABLELM,+ LLM_ARCH_QWEN,LLM_ARCH_UNKNOWN,};@@ -208,6 +232,7 @@ static std::map<llm_arch, std::string> LLM_ARCH_NAMES = {{ LLM_ARCH_REFACT, "refact" },{ LLM_ARCH_BLOOM, "bloom" },{ LLM_ARCH_STABLELM, "stablelm" },+ { LLM_ARCH_QWEN, "qwen" },};enum llm_kv {@@ -228,6 +253,8 @@ enum llm_kv {LLM_KV_FEED_FORWARD_LENGTH,LLM_KV_USE_PARALLEL_RESIDUAL,LLM_KV_TENSOR_DATA_LAYOUT,+ LLM_KV_EXPERT_COUNT,+ LLM_KV_EXPERT_USED_COUNT,LLM_KV_ATTENTION_HEAD_COUNT,LLM_KV_ATTENTION_HEAD_COUNT_KV,@@ -254,6 +281,8 @@ enum llm_kv {LLM_KV_TOKENIZER_UNK_ID,LLM_KV_TOKENIZER_SEP_ID,LLM_KV_TOKENIZER_PAD_ID,+ LLM_KV_TOKENIZER_ADD_BOS,+ LLM_KV_TOKENIZER_ADD_EOS,LLM_KV_TOKENIZER_HF_JSON,LLM_KV_TOKENIZER_RWKV,};@@ -276,6 +305,8 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {{ LLM_KV_FEED_FORWARD_LENGTH, "%s.feed_forward_length" },{ LLM_KV_USE_PARALLEL_RESIDUAL, "%s.use_parallel_residual" },{ LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" },+ { LLM_KV_EXPERT_COUNT, "%s.expert_count" },+ { LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" },{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },@@ -302,6 +333,8 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {{ LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },{ LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },{ LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },+ { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },+ { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },};@@ -331,10 +364,14 @@ enum llm_tensor {LLM_TENSOR_ATTN_NORM,LLM_TENSOR_ATTN_NORM_2,LLM_TENSOR_ATTN_ROT_EMBD,+ LLM_TENSOR_FFN_GATE_INP,+ LLM_TENSOR_FFN_NORM,LLM_TENSOR_FFN_GATE,LLM_TENSOR_FFN_DOWN,LLM_TENSOR_FFN_UP,- LLM_TENSOR_FFN_NORM,+ LLM_TENSOR_FFN_DOWN_EXP,+ LLM_TENSOR_FFN_GATE_EXP,+ LLM_TENSOR_FFN_UP_EXP,LLM_TENSOR_ATTN_Q_NORM,LLM_TENSOR_ATTN_K_NORM,};@@ -353,10 +390,14 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },{ LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" },+ { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" },+ { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" },},},{@@ -514,6 +555,22 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_QWEN,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_UNKNOWN,@@ -562,27 +619,16 @@ struct LLM_TN {std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const {return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix;}++ std::string operator()(llm_tensor tensor, const std::string & suffix, int bid, int xid) const {+ return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid, xid) + "." + suffix;+ }};//// gguf helpers//-#define GGUF_GET_KEY(ctx, dst, func, type, req, key) \-do { \- const std::string skey(key); \- const int kid = gguf_find_key(ctx, skey.c_str()); \- if (kid >= 0) { \- enum gguf_type ktype = gguf_get_kv_type(ctx, kid); \- if (ktype != (type)) { \- ThrowRuntimeError(format("key %s has wrong type: %s", skey.c_str(), gguf_type_name(ktype))); \- } \- (dst) = func(ctx, kid); \- } else if (req) { \- ThrowRuntimeError(format("key not found in model: %s", skey.c_str())); \- } \-} while (0)-static std::map<int8_t, std::string> LLAMA_ROPE_SCALING_TYPES = {{ LLAMA_ROPE_SCALING_NONE, "none" },{ LLAMA_ROPE_SCALING_LINEAR, "linear" },@@ -599,6 +645,60 @@ static int8_t llama_rope_scaling_type_from_string(const std::string & name) {return LLAMA_ROPE_SCALING_UNSPECIFIED;}+static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {+ switch (type) {+ case GGUF_TYPE_UINT8: return std::to_string(((const uint8_t *)data)[i]);+ case GGUF_TYPE_INT8: return std::to_string(((const int8_t *)data)[i]);+ case GGUF_TYPE_UINT16: return std::to_string(((const uint16_t *)data)[i]);+ case GGUF_TYPE_INT16: return std::to_string(((const int16_t *)data)[i]);+ case GGUF_TYPE_UINT32: return std::to_string(((const uint32_t *)data)[i]);+ case GGUF_TYPE_INT32: return std::to_string(((const int32_t *)data)[i]);+ case GGUF_TYPE_UINT64: return std::to_string(((const uint64_t *)data)[i]);+ case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);+ case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);+ case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);+ case GGUF_TYPE_BOOL: return ((const bool *)data)[i] ? "true" : "false";+ default: return format("unknown type %d", type);+ }+}++static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {+ const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);++ switch (type) {+ case GGUF_TYPE_STRING:+ return gguf_get_val_str(ctx_gguf, i);+ case GGUF_TYPE_ARRAY:+ {+ const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);+ int arr_n = gguf_get_arr_n(ctx_gguf, i);+ const void * data = gguf_get_arr_data(ctx_gguf, i);+ std::stringstream ss;+ ss << "[";+ for (int j = 0; j < arr_n; j++) {+ if (arr_type == GGUF_TYPE_STRING) {+ std::string val = gguf_get_arr_str(ctx_gguf, i, j);+ // escape quotes+ replace_all(val, "\\", "\\\\");+ replace_all(val, "\"", "\\\"");+ ss << '"' << val << '"';+ } else if (arr_type == GGUF_TYPE_ARRAY) {+ ss << "???";+ } else {+ ss << gguf_data_to_str(arr_type, data, j);+ }+ if (j < arr_n - 1) {+ ss << ", ";+ }+ }+ ss << "]";+ return ss.str();+ }+ default:+ return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);+ }+}+//// ggml helpers//@@ -626,6 +726,9 @@ inline void * llama_host_malloc(size_t n) {} else {return malloc(n);}+#if GGML_USE_CPU_HBM+#error fix me+#endif}inline void llama_host_free(void * ptr) {@@ -636,6 +739,9 @@ inline void llama_host_free(void * ptr) {} else {return free(ptr);}+#if GGML_USE_CPU_HBM+#error fix me+#endif}struct llama_buffer {@@ -647,7 +753,7 @@ struct llama_buffer {bool fallback = false;void resize(size_t n) {- release();+ llama_host_free(data);data = llama_host_malloc(n);if (!data) {@@ -661,7 +767,7 @@ struct llama_buffer {size = n;}- void release(void) {+ ~llama_buffer() {if (data) {if (fallback) { // NOLINTfree(data);@@ -669,11 +775,8 @@ struct llama_buffer {llama_host_free(data);}}- data = NULL;- }- ~llama_buffer() {- release();+ data = NULL;}};@@ -982,9 +1085,9 @@ enum e_model {MODEL_70B,};-static const size_t kB = 1024;-static const size_t MB = 1024*kB;-static const size_t GB = 1024*MB;+static const size_t kiB = 1024;+static const size_t MiB = 1024*kiB;+static const size_t GiB = 1024*MiB;struct llama_hparams {bool vocab_only;@@ -996,6 +1099,8 @@ struct llama_hparams {uint32_t n_layer;uint32_t n_rot;uint32_t n_ff;+ uint32_t n_expert = 0;+ uint32_t n_expert_used = 0;float f_norm_eps;float f_norm_rms_eps;@@ -1010,15 +1115,18 @@ struct llama_hparams {float f_max_alibi_bias;bool operator!=(const llama_hparams & other) const {- if (this->vocab_only != other.vocab_only) return true;- if (this->n_vocab != other.n_vocab) return true;- if (this->n_ctx_train != other.n_ctx_train) return true;- if (this->n_embd != other.n_embd) return true;- if (this->n_head != other.n_head) return true;- if (this->n_head_kv != other.n_head_kv) return true;- if (this->n_layer != other.n_layer) return true;- if (this->n_rot != other.n_rot) return true;- if (this->n_ff != other.n_ff) return true;+ if (this->vocab_only != other.vocab_only) return true;+ if (this->n_vocab != other.n_vocab) return true;+ if (this->n_ctx_train != other.n_ctx_train) return true;+ if (this->n_embd != other.n_embd) return true;+ if (this->n_head != other.n_head) return true;+ if (this->n_head_kv != other.n_head_kv) return true;+ if (this->n_layer != other.n_layer) return true;+ if (this->n_rot != other.n_rot) return true;+ if (this->n_ff != other.n_ff) return true;+ if (this->n_expert != other.n_expert) return true;+ if (this->n_expert_used != other.n_expert_used) return true;+if (this->rope_finetuned != other.rope_finetuned) return true;if (this->n_yarn_orig_ctx != other.n_yarn_orig_ctx) return true;@@ -1063,6 +1171,7 @@ struct llama_cparams {float yarn_beta_slow;bool mul_mat_q;+ bool offload_kqv;};struct llama_layer {@@ -1099,6 +1208,12 @@ struct llama_layer {struct ggml_tensor * ffn_down; // w2struct ggml_tensor * ffn_up; // w3+ // ff MoE+ struct ggml_tensor * ffn_gate_inp;+ struct ggml_tensor * ffn_gate_exp[LLAMA_MAX_EXPERTS];+ struct ggml_tensor * ffn_down_exp[LLAMA_MAX_EXPERTS];+ struct ggml_tensor * ffn_up_exp [LLAMA_MAX_EXPERTS];+// ff biasstruct ggml_tensor * ffn_down_b; // b2struct ggml_tensor * ffn_up_b; // b3@@ -1124,14 +1239,15 @@ struct llama_kv_cache {// cannot be freely changed after a slot has been allocated.uint32_t head = 0;uint32_t size = 0;+ uint32_t used = 0; // used cells (i.e. at least one seq_id)// computed before each graph builduint32_t n = 0;std::vector<llama_kv_cell> cells;- struct ggml_tensor * k = NULL;- struct ggml_tensor * v = NULL;+ std::vector<struct ggml_tensor *> k_l; // per layer+ std::vector<struct ggml_tensor *> v_l;struct ggml_context * ctx = NULL;@@ -1143,8 +1259,10 @@ struct llama_kv_cache {}if (!ggml_metal_supported() && ggml_cublas_loaded()) {- ggml_cuda_free_data(k);- ggml_cuda_free_data(v);+ for (size_t i = 0; i < k_l.size(); ++i) {+ ggml_cuda_free_data(k_l[i]);+ ggml_cuda_free_data(v_l[i]);+ }}}};@@ -1176,6 +1294,9 @@ struct llama_vocab {id special_sep_id = -1;id special_pad_id = -1;+ int special_add_bos = -1; // -1 unknown, 1 add, 0 don't add.+ int special_add_eos = -1; // -1 unknown, 1 add, 0 don't add.+id linefeed_id = 13;id special_prefix_id = 32007;id special_middle_id = 32009;@@ -1220,6 +1341,9 @@ struct llama_model {int n_gpu_layers;+ // gguf metadata+ std::unordered_map<std::string, std::string> gguf_kv;+// contextstruct ggml_context * ctx = NULL;@@ -1321,9 +1445,11 @@ struct llama_context {static bool llama_kv_cache_init(const struct llama_hparams & hparams,struct llama_kv_cache & cache,- ggml_type wtype,+ ggml_type ktype,+ ggml_type vtype,uint32_t n_ctx,- int n_gpu_layers) {+ int n_gpu_layers,+ bool offload) {const uint32_t n_embd = hparams.n_embd_gqa();const uint32_t n_layer = hparams.n_layer;@@ -1334,11 +1460,12 @@ static bool llama_kv_cache_init(cache.head = 0;cache.size = n_ctx;+ cache.used = 0;cache.cells.clear();cache.cells.resize(n_ctx);- cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*ggml_tensor_overhead());+ cache.buf.resize(n_elements*(ggml_type_sizef(ktype) + ggml_type_sizef(vtype)) + 2u*n_layer*ggml_tensor_overhead());memset(cache.buf.data, 0, cache.buf.size);struct ggml_init_params params;@@ -1348,36 +1475,45 @@ static bool llama_kv_cache_init(cache.ctx = ggml_init(params);+ size_t vram_kv_cache = 0;+if (!cache.ctx) {LLAMA_LOG_ERROR("%s: failed to allocate memory for kv cache\n", __func__);return false;}- cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);- cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);- ggml_set_name(cache.k, "cache_k");- ggml_set_name(cache.v, "cache_v");+ cache.k_l.reserve(n_layer);+ cache.v_l.reserve(n_layer);- (void) n_gpu_layers;+ const int i_gpu_start = (int) n_layer - n_gpu_layers; GGML_UNUSED(i_gpu_start);- if (!ggml_metal_supported() && ggml_cublas_loaded()) {- size_t vram_kv_cache = 0;+ GGML_UNUSED(offload);- if (n_gpu_layers > (int)n_layer + 1) {- ggml_cuda_assign_buffers_no_scratch(cache.v);- LLAMA_LOG_INFO("%s: offloading v cache to GPU\n", __func__);- vram_kv_cache += ggml_nbytes(cache.v);- }- if (n_gpu_layers > (int)n_layer + 2) {- ggml_cuda_assign_buffers_no_scratch(cache.k);- LLAMA_LOG_INFO("%s: offloading k cache to GPU\n", __func__);- vram_kv_cache += ggml_nbytes(cache.k);+ for (int i = 0; i < (int) n_layer; i++) {+ ggml_tensor * k = ggml_new_tensor_1d(cache.ctx, ktype, n_embd*n_ctx);+ ggml_tensor * v = ggml_new_tensor_1d(cache.ctx, vtype, n_embd*n_ctx);+ ggml_format_name(k, "cache_k_l%d", i);+ ggml_format_name(v, "cache_v_l%d", i);+ cache.k_l.push_back(k);+ cache.v_l.push_back(v);+ if (!ggml_metal_supported() && ggml_cublas_loaded()) {+ if (i >= i_gpu_start) {+ if (offload) {+ ggml_cuda_assign_buffers_no_scratch(k);+ vram_kv_cache += ggml_nbytes(k);+ ggml_cuda_assign_buffers_no_scratch(v);+ vram_kv_cache += ggml_nbytes(v);+ }}- if (vram_kv_cache > 0) {- LLAMA_LOG_INFO("%s: VRAM kv self = %.2f MB\n", __func__, vram_kv_cache / 1024.0 / 1024.0);}}+ if (vram_kv_cache > 0) {+ LLAMA_LOG_INFO("%s: VRAM kv self = %.2f MB\n", __func__, vram_kv_cache / 1024.0 / 1024.0);+ }++ GGML_UNUSED(n_gpu_layers);+return true;}@@ -1433,6 +1569,8 @@ static bool llama_kv_cache_find_slot(}}+ cache.used += n_tokens;+return true;}@@ -1453,6 +1591,7 @@ static void llama_kv_cache_clear(struct llama_kv_cache & cache) {cache.cells[i].seq_id.clear();}cache.head = 0;+ cache.used = 0;}static void llama_kv_cache_seq_rm(@@ -1475,6 +1614,9 @@ static void llama_kv_cache_seq_rm(continue;}if (cache.cells[i].seq_id.empty()) {+ // keep count of the number of used cells+ if (cache.cells[i].pos >= 0) cache.used--;+cache.cells[i].pos = -1;if (new_head == cache.size) new_head = i;}@@ -1482,7 +1624,7 @@ static void llama_kv_cache_seq_rm(}// If we freed up a slot, set head to it so searching can start there.- if (new_head != cache.size) cache.head = new_head;+ if (new_head != cache.size && new_head < cache.head) cache.head = new_head;}static void llama_kv_cache_seq_cp(@@ -1508,6 +1650,7 @@ static void llama_kv_cache_seq_keep(struct llama_kv_cache & cache, llama_seq_idfor (uint32_t i = 0; i < cache.size; ++i) {if (!cache.cells[i].has_seq_id(seq_id)) {+ if (cache.cells[i].pos >= 0) cache.used--;cache.cells[i].pos = -1;cache.cells[i].seq_id.clear();if (new_head == cache.size) new_head = i;@@ -1518,7 +1661,7 @@ static void llama_kv_cache_seq_keep(struct llama_kv_cache & cache, llama_seq_id}// If we freed up a slot, set head to it so searching can start there.- if (new_head != cache.size) cache.head = new_head;+ if (new_head != cache.size && new_head < cache.head) cache.head = new_head;}static void llama_kv_cache_seq_shift(@@ -1539,6 +1682,7 @@ static void llama_kv_cache_seq_shift(cache.cells[i].delta += delta;if (cache.cells[i].pos < 0) {+ if (!cache.cells[i].seq_id.empty()) cache.used--;cache.cells[i].pos = -1;cache.cells[i].seq_id.clear();if (new_head == cache.size) new_head = i;@@ -1589,6 +1733,169 @@ static std::string llama_format_tensor_shape(const struct ggml_tensor * t) {return buf;}+namespace GGUFMeta {+ template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int)>+ struct GKV_Base_Type {+ static constexpr gguf_type gt = gt_;++ static T getter(const gguf_context * ctx, const int kid) {+ return gfun(ctx, kid);+ }+ };++ template<typename T> struct GKV_Base;++ template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};+ template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};+ template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};+ template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};+ template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};+ template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};+ template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};+ template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};+ template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};+ template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};+ template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};+ template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};++ template<> struct GKV_Base<std::string> {+ static constexpr gguf_type gt = GGUF_TYPE_STRING;++ static std::string getter(const gguf_context * ctx, const int kid) {+ return gguf_get_val_str(ctx, kid);+ }+ };++ struct ArrayInfo{+ const gguf_type gt;+ const size_t length;+ const void * data;+ };++ template<> struct GKV_Base<ArrayInfo> {+ public:+ static constexpr gguf_type gt = GGUF_TYPE_ARRAY;+ static ArrayInfo getter(const gguf_context *ctx, const int k) {+ return ArrayInfo {+ gguf_get_arr_type(ctx, k),+ size_t(gguf_get_arr_n(ctx, k)),+ gguf_get_arr_data(ctx, k),+ };+ }+ };++ template<typename T>+ class GKV: public GKV_Base<T> {+ GKV() = delete;++ public:+ static T get_kv(const gguf_context * ctx, const int k) {+ const enum gguf_type kt = gguf_get_kv_type(ctx, k);++ if (kt != GKV::gt) {+ ThrowRuntimeError(format("key %s has wrong type %s but expected type %s",+ gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));+ }+ return GKV::getter(ctx, k);+ }++ static const char * override_type_to_str(const llama_model_kv_override_type ty) {+ switch (ty) {+ case LLAMA_KV_OVERRIDE_BOOL: return "bool";+ case LLAMA_KV_OVERRIDE_INT: return "int";+ case LLAMA_KV_OVERRIDE_FLOAT: return "float";+ }+ return "unknown";+ }++ static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override *override) {+ if (!override) { return false; }+ if (override->tag == expected_type) {+ LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",+ __func__, override_type_to_str(override->tag), override->key);+ switch (override->tag) {+ case LLAMA_KV_OVERRIDE_BOOL: {+ printf("%s\n", override->bool_value ? "true" : "false");+ } break;+ case LLAMA_KV_OVERRIDE_INT: {+ printf("%" PRId64 "\n", override->int_value);+ } break;+ case LLAMA_KV_OVERRIDE_FLOAT: {+ printf("%.6f\n", override->float_value);+ } break;+ default:+ // Shouldn't be possible to end up here, but just in case...+ ThrowRuntimeError(+ format("Unsupported attempt to override %s type for metadata key %s\n",+ override_type_to_str(override->tag), override->key));+ }+ return true;+ }+ LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",+ __func__, override->key, override_type_to_str(expected_type), override_type_to_str(override->tag));+ return false;+ }++ template<typename OT>+ static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type+ try_override(OT & target, const struct llama_model_kv_override *override) {+ if (validate_override(LLAMA_KV_OVERRIDE_BOOL, override)) {+ target = override->bool_value;+ return true;+ }+ return true;+ }++ template<typename OT>+ static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type+ try_override(OT & target, const struct llama_model_kv_override *override) {+ if (validate_override(LLAMA_KV_OVERRIDE_INT, override)) {+ target = override->int_value;+ return true;+ }+ return false;+ }++ template<typename OT>+ static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type+ try_override(T & target, const struct llama_model_kv_override *override) {+ if (validate_override(LLAMA_KV_OVERRIDE_FLOAT, override)) {+ target = override->float_value;+ return true;+ }+ return false;+ }++ template<typename OT>+ static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type+ try_override(T & target, const struct llama_model_kv_override *override) {+ (void)target;+ (void)override;+ if (!override) { return false; }+ // Currently, we should never end up here so it would be a bug if we do.+ ThrowRuntimeError(format("Unsupported attempt to override string type for metadata key %s\n",+ override ? override->key : "NULL"));+ }++ static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override *override = nullptr) {+ if (try_override<T>(target, override)) {+ return true;+ }+ if (k < 0) { return false; }+ target = get_kv(ctx, k);+ return true;+ }++ static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override *override = nullptr) {+ return set(ctx, gguf_find_key(ctx, key), target, override);+ }++ static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override *override = nullptr) {+ return set(ctx, key.c_str(), target, override);+ }+ };+}+struct llama_model_loader {int n_kv = 0;int n_tensors = 0;@@ -1604,22 +1911,35 @@ struct llama_model_loader {llama_fver fver;std::unique_ptr<llama_mmap> mapping;+ std::unordered_map<std::string, struct llama_model_kv_override> kv_overrides;struct gguf_context * ctx_gguf = NULL;struct ggml_context * ctx_meta = NULL;- llama_model_loader(const std::string & fname, bool use_mmap) : file(fname.c_str(), "rbe") {+ std::string arch_name;+ LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);++ llama_model_loader(const std::string & fname, bool use_mmap, const struct llama_model_kv_override * param_overrides_p) : file(fname.c_str(), "rbe") {struct gguf_init_params params = {/*.no_alloc = */ true,/*.ctx = */ &ctx_meta,};+ if (param_overrides_p != nullptr) {+ for (const struct llama_model_kv_override *p = param_overrides_p; p->key[0] != 0; p++) {+ kv_overrides.insert({std::string(p->key), *p});+ }+ }+ctx_gguf = gguf_init_from_file(file.file, params);if (!ctx_gguf) {ThrowRuntimeError(format("%s: failed to load model from %s\n", __func__, fname.c_str()));}llamafile_seek(file.file, 0, SEEK_SET);+ get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);+ llm_kv = LLM_KV(llm_arch_from_string(arch_name));+n_kv = gguf_get_n_kv(ctx_gguf);n_tensors = gguf_get_n_tensors(ctx_gguf);@@ -1671,10 +1991,10 @@ struct llama_model_loader {case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;default:- {- LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));- ftype = LLAMA_FTYPE_ALL_F32;- } break;+ {+ LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));+ ftype = LLAMA_FTYPE_ALL_F32;+ } break;}// this is a way to mark that we have "guessed" the file type@@ -1687,11 +2007,23 @@ struct llama_model_loader {}}+ LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);for (int i = 0; i < n_kv; i++) {- const char * name = gguf_get_key(ctx_gguf, i);- const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);+ const char * name = gguf_get_key(ctx_gguf, i);+ const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);+ const std::string type_name =+ type == GGUF_TYPE_ARRAY+ ? format("%s[%s,%d]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(ctx_gguf, i)), gguf_get_arr_n(ctx_gguf, i))+ : gguf_type_name(type);++ std::string value = gguf_kv_to_str(ctx_gguf, i);+ const size_t MAX_VALUE_LEN = 40;+ if (value.size() > MAX_VALUE_LEN) {+ value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());+ }+ replace_all(value, "\n", "\\n");- LLAMA_LOG_INFO("%s: - kv %3d: %42s %-8s\n", __func__, i, name, gguf_type_name(type));+ LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());}// print type counts@@ -1721,19 +2053,59 @@ struct llama_model_loader {}}- std::string get_arch_name() const {- const auto kv = LLM_KV(LLM_ARCH_UNKNOWN);+ template<typename T>+ typename std::enable_if<std::is_integral<T>::value, bool>::type+ get_arr_n(const std::string & key, T & result, const bool required = true) {+ const int kid = gguf_find_key(ctx_gguf, key.c_str());++ if (kid < 0) {+ if (required) {+ ThrowRuntimeError(format("key not found in model: %s", key.c_str()));+ }+ return false;+ }- std::string arch_name;- GGUF_GET_KEY(ctx_gguf, arch_name, gguf_get_val_str, GGUF_TYPE_STRING, false, kv(LLM_KV_GENERAL_ARCHITECTURE));+ struct GGUFMeta::ArrayInfo arr_info =+ GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx_gguf, kid);++ result = arr_info.length;+ return true;+ }++ template<typename T>+ typename std::enable_if<std::is_integral<T>::value, bool>::type+ get_arr_n(const enum llm_kv kid, T & result, const bool required = true) {+ return get_arr_n(llm_kv(kid), result, required);+ }++ template<typename T>+ bool get_key(const std::string & key, T & result, const bool required = true) {+ auto it = kv_overrides.find(key);++ const struct llama_model_kv_override * override =+ it != kv_overrides.end() ? &it->second : nullptr;++ const bool found = GGUFMeta::GKV<T>::set(ctx_gguf, key, result, override);++ if (required && !found) {+ ThrowRuntimeError(format("key not found in model: %s", key.c_str()));+ }++ return found;+ }++ template<typename T>+ bool get_key(const enum llm_kv kid, T & result, const bool required = true) {+ return get_key(llm_kv(kid), result, required);+ }++ std::string get_arch_name() const {return arch_name;}enum llm_arch get_arch() const {- const std::string arch_name = get_arch_name();-- return llm_arch_from_string(arch_name);+ return llm_kv.arch;}const char * get_tensor_name(int i) const {@@ -1773,11 +2145,11 @@ struct llama_model_loader {return tensor;}- struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector<int64_t> & ne, ggml_backend_type backend, bool optional = false) {+ struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector<int64_t> & ne, ggml_backend_type backend, bool required = true) {struct ggml_tensor * cur = ggml_get_tensor(ctx_meta, name.c_str());if (cur == NULL) {- if (optional) {+ if (!required) {return NULL;}ThrowRuntimeError(format("%s: tensor '%s' not found", __func__, name.c_str()));@@ -1888,12 +2260,21 @@ struct llama_model_loader {case GGML_BACKEND_GPU_SPLIT:// old code://ggml_cuda_transform_tensor(lt.data, lt.ggml_tensor);+// TODO: test if this works !!ggml_cuda_transform_tensor(cur->data, cur);if (!use_mmap) {free(cur->data);}break;+#if defined(GGML_USE_CLBLAST)+ case GGML_BACKEND_GPU:+ ggml_cl_transform_tensor(cur->data, cur);+ if (!use_mmap) {+ free(cur->data);+ }+ break;+#endifdefault:continue;}@@ -1973,49 +2354,66 @@ static void llm_load_arch(llama_model_loader & ml, llama_model & model) {static void llm_load_hparams(llama_model_loader & ml,llama_model & model) {- struct gguf_context * ctx = ml.ctx_gguf;-- const auto kv = LLM_KV(model.arch);-auto & hparams = model.hparams;+ const gguf_context * ctx = ml.ctx_gguf;++ // get metadata as string+ for (int i = 0; i < gguf_get_n_kv(ctx); i++) {+ enum gguf_type type = gguf_get_kv_type(ctx, i);+ if (type == GGUF_TYPE_ARRAY) {+ continue;+ }+ const char * name = gguf_get_key(ctx, i);+ const std::string value = gguf_kv_to_str(ctx, i);+ model.gguf_kv.emplace(name, value);+ }// get general kv- GGUF_GET_KEY(ctx, model.name, gguf_get_val_str, GGUF_TYPE_STRING, false, kv(LLM_KV_GENERAL_NAME));+ ml.get_key(LLM_KV_GENERAL_NAME, model.name, false);// get hparams kv- GGUF_GET_KEY(ctx, hparams.n_vocab, gguf_get_arr_n, GGUF_TYPE_ARRAY, true, kv(LLM_KV_TOKENIZER_LIST));- GGUF_GET_KEY(ctx, hparams.n_ctx_train, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_CONTEXT_LENGTH));- GGUF_GET_KEY(ctx, hparams.n_embd, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_EMBEDDING_LENGTH));- GGUF_GET_KEY(ctx, hparams.n_ff, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_FEED_FORWARD_LENGTH));- GGUF_GET_KEY(ctx, hparams.n_head, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_ATTENTION_HEAD_COUNT));- GGUF_GET_KEY(ctx, hparams.n_layer, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_BLOCK_COUNT));+ ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);+ ml.get_key (LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);+ ml.get_key (LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);+ ml.get_key (LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff);+ ml.get_key (LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head);+ ml.get_key (LLM_KV_BLOCK_COUNT, hparams.n_layer);+ ml.get_key (LLM_KV_EXPERT_COUNT, hparams.n_expert, false);+ ml.get_key (LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);++ GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);+ GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);+ if (hparams.n_expert > 0) {+ GGML_ASSERT(hparams.n_expert_used > 0);+ } else {+ GGML_ASSERT(hparams.n_expert_used == 0);+ }// n_head_kv is optional, default to n_headhparams.n_head_kv = hparams.n_head;- GGUF_GET_KEY(ctx, hparams.n_head_kv, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_ATTENTION_HEAD_COUNT_KV));+ ml.get_key(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv, false);- hparams.rope_finetuned = false;- GGUF_GET_KEY(ctx, hparams.rope_finetuned, gguf_get_val_bool, GGUF_TYPE_BOOL, false,- kv(LLM_KV_ROPE_SCALING_FINETUNED));+ bool rope_finetuned = false;+ ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);+ hparams.rope_finetuned = rope_finetuned;hparams.n_yarn_orig_ctx = hparams.n_ctx_train;- GGUF_GET_KEY(ctx, hparams.n_yarn_orig_ctx, gguf_get_val_u32, GGUF_TYPE_UINT32, false,- kv(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN));+ ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_yarn_orig_ctx, false);// rope_freq_base (optional)hparams.rope_freq_base_train = 10000.0f;- GGUF_GET_KEY(ctx, hparams.rope_freq_base_train, gguf_get_val_f32, GGUF_TYPE_FLOAT32, false, kv(LLM_KV_ROPE_FREQ_BASE));+ ml.get_key(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train, false);std::string rope_scaling("linear");- GGUF_GET_KEY(ctx, rope_scaling, gguf_get_val_str, GGUF_TYPE_STRING, false, kv(LLM_KV_ROPE_SCALING_TYPE));+ ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false);hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_UNSPECIFIED);// rope_freq_scale (inverse of the kv) is optionalfloat ropescale = 0.0f;- GGUF_GET_KEY(ctx, ropescale, gguf_get_val_f32, GGUF_TYPE_FLOAT32, false, kv(LLM_KV_ROPE_SCALING_FACTOR));- if (ropescale == 0.0f) { // try the old key name- GGUF_GET_KEY(ctx, ropescale, gguf_get_val_f32, GGUF_TYPE_FLOAT32, false, kv(LLM_KV_ROPE_SCALE_LINEAR));+ if (!ml.get_key(LLM_KV_ROPE_SCALING_FACTOR, ropescale, false)) {+ // try the old key name+ ml.get_key(LLM_KV_ROPE_SCALE_LINEAR, ropescale, false);}hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;@@ -2023,7 +2421,7 @@ static void llm_load_hparams({hparams.n_rot = hparams.n_embd / hparams.n_head;- GGUF_GET_KEY(ctx, hparams.n_rot, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_ROPE_DIMENSION_COUNT));+ ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot, false);if (model.arch == LLM_ARCH_LLAMA || model.arch == LLM_ARCH_FALCON) {if (hparams.n_rot != hparams.n_embd / hparams.n_head) {@@ -2038,7 +2436,7 @@ static void llm_load_hparams(switch (model.arch) {case LLM_ARCH_LLAMA:{- GGUF_GET_KEY(ctx, hparams.f_norm_rms_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer) {case 26: model.type = e_model::MODEL_3B; break;@@ -2052,7 +2450,7 @@ static void llm_load_hparams(} break;case LLM_ARCH_FALCON:{- GGUF_GET_KEY(ctx, hparams.f_norm_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);switch (hparams.n_layer) {case 32: model.type = e_model::MODEL_7B; break;@@ -2062,7 +2460,7 @@ static void llm_load_hparams(} break;case LLM_ARCH_BAICHUAN:{- GGUF_GET_KEY(ctx, hparams.f_norm_rms_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer) {case 32: model.type = e_model::MODEL_7B; break;case 40: model.type = e_model::MODEL_13B; break;@@ -2071,7 +2469,7 @@ static void llm_load_hparams(} break;case LLM_ARCH_STARCODER:{- GGUF_GET_KEY(ctx, hparams.f_norm_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);switch (hparams.n_layer) {case 24: model.type = e_model::MODEL_1B; break;case 36: model.type = e_model::MODEL_3B; break;@@ -2082,7 +2480,7 @@ static void llm_load_hparams(} break;case LLM_ARCH_PERSIMMON:{- GGUF_GET_KEY(ctx, hparams.f_norm_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);switch (hparams.n_layer) {case 36: model.type = e_model::MODEL_8B; break;default: model.type = e_model::MODEL_UNKNOWN;@@ -2090,7 +2488,7 @@ static void llm_load_hparams(} break;case LLM_ARCH_REFACT:{- GGUF_GET_KEY(ctx, hparams.f_norm_rms_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer) {case 32: model.type = e_model::MODEL_1B; break;default: model.type = e_model::MODEL_UNKNOWN;@@ -2098,7 +2496,7 @@ static void llm_load_hparams(} break;case LLM_ARCH_BLOOM:{- GGUF_GET_KEY(ctx, hparams.f_norm_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);switch (hparams.n_layer) {case 24: model.type = e_model::MODEL_1B; break;@@ -2113,9 +2511,9 @@ static void llm_load_hparams({hparams.f_clamp_kqv = 0.0f;- GGUF_GET_KEY(ctx, hparams.f_norm_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_EPS));- GGUF_GET_KEY(ctx, hparams.f_clamp_kqv, gguf_get_val_f32, GGUF_TYPE_FLOAT32, false, kv(LLM_KV_ATTENTION_CLAMP_KQV));- GGUF_GET_KEY(ctx, hparams.f_max_alibi_bias, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv, false);+ ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);switch (hparams.n_layer) {case 32: model.type = e_model::MODEL_7B; break;@@ -2125,13 +2523,23 @@ static void llm_load_hparams(} break;case LLM_ARCH_STABLELM:{- GGUF_GET_KEY(ctx, hparams.f_norm_eps, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, kv(LLM_KV_ATTENTION_LAYERNORM_EPS));+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);switch (hparams.n_layer) {case 32: model.type = e_model::MODEL_3B; break;default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_QWEN:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);++ switch (hparams.n_layer) {+ case 32: model.type = e_model::MODEL_7B; break;+ case 40: model.type = e_model::MODEL_13B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;default: (void)0;}@@ -2173,7 +2581,7 @@ static void llm_load_vocab({std::string tokenizer_name;- GGUF_GET_KEY(ctx, tokenizer_name, gguf_get_val_str, GGUF_TYPE_STRING, true, kv(LLM_KV_TOKENIZER_MODEL));+ ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_name);if (tokenizer_name == "llama") {vocab.type = LLAMA_VOCAB_TYPE_SPM;@@ -2197,7 +2605,7 @@ static void llm_load_vocab(for (int i = 0; i < n_merges; i++) {const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);- // GGML_ASSERT(codepoints_from_utf8(word).size() > 0);+ GGML_ASSERT(codepoints_from_utf8(word).size() > 0);std::string first;std::string second;@@ -2232,7 +2640,7 @@ static void llm_load_vocab(for (uint32_t i = 0; i < n_vocab; i++) {std::string word = gguf_get_arr_str(ctx, token_idx, i);- // GGML_ASSERT(codepoints_from_utf8(word).size() > 0);+ GGML_ASSERT(codepoints_from_utf8(word).size() > 0);vocab.token_to_id[word] = i;@@ -2263,16 +2671,30 @@ static void llm_load_vocab(};for (const auto & it : special_token_types) {const std::string & key = kv(std::get<0>(it));- int32_t & id = std::get<1>(it), old_id = id;+ int32_t & id = std::get<1>(it);++ uint32_t new_id;+ if (!ml.get_key(std::get<0>(it), new_id, false)) {+ continue;+ }+ if (new_id >= vocab.id_to_token.size()) {+ LLAMA_LOG_WARN("%s: bad special token: '%s' = %ud, using default id %d\n",+ __func__, key.c_str(), new_id, id);+ } else {+ id = new_id;+ }++ }++ // Handle add_bos_token and add_eos_token+ {+ bool temp = true;- GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, key);- // Must be >= -1 and < vocab size. Since the key is unsigned, -1- // can only come from the default value, so there's no point in- // validating that.- if (size_t(id + 1) > vocab.id_to_token.size()) {- LLAMA_LOG_WARN("%s: bad special token: '%s' = %d, using default id %d\n",- __func__, key.c_str(), id, old_id);- id = old_id;+ if (ml.get_key(LLM_KV_TOKENIZER_ADD_BOS, temp, false)) {+ vocab.special_add_bos = int(temp);+ }+ if (ml.get_key(LLM_KV_TOKENIZER_ADD_EOS, temp, false)) {+ vocab.special_add_eos = int(temp);}}}@@ -2397,6 +2819,8 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv);LLAMA_LOG_INFO("%s: f_max_alibi_bias = %.1e\n", __func__, hparams.f_max_alibi_bias);LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff);+ LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert);+ LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used);LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type.c_str());LLAMA_LOG_INFO("%s: freq_base_train = %.1f\n", __func__, hparams.rope_freq_base_train);LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train);@@ -2405,22 +2829,22 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type));LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str());LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, ml.n_elements*1e-9);- if (ml.n_bytes < GB) {- LLAMA_LOG_INFO("%s: model size = %.2f MiB (%.2f BPW) \n", __func__, ml.n_bytes/1024.0/1024.0, ml.n_bytes*8.0/ml.n_elements);+ if (ml.n_bytes < GiB) {+ LLAMA_LOG_INFO("%s: model size = %.2f MiB (%.2f BPW) \n", __func__, ml.n_bytes/1024.0/1024.0, ml.n_bytes*8.0/ml.n_elements);} else {LLAMA_LOG_INFO("%s: model size = %.2f GiB (%.2f BPW) \n", __func__, ml.n_bytes/1024.0/1024.0/1024.0, ml.n_bytes*8.0/ml.n_elements);}// general kv- LLAMA_LOG_INFO("%s: general.name = %s\n", __func__, model.name.c_str());+ LLAMA_LOG_INFO("%s: general.name = %s\n", __func__, model.name.c_str());// special tokens- if (vocab.special_bos_id != -1) { LLAMA_LOG_INFO( "%s: BOS token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].text.c_str() ); }- if (vocab.special_eos_id != -1) { LLAMA_LOG_INFO( "%s: EOS token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].text.c_str() ); }- if (vocab.special_unk_id != -1) { LLAMA_LOG_INFO( "%s: UNK token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].text.c_str() ); }- if (vocab.special_sep_id != -1) { LLAMA_LOG_INFO( "%s: SEP token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].text.c_str() ); }- if (vocab.special_pad_id != -1) { LLAMA_LOG_INFO( "%s: PAD token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].text.c_str() ); }- if (vocab.linefeed_id != -1) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }+ if (vocab.special_bos_id != -1) { LLAMA_LOG_INFO( "%s: BOS token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].text.c_str() ); }+ if (vocab.special_eos_id != -1) { LLAMA_LOG_INFO( "%s: EOS token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].text.c_str() ); }+ if (vocab.special_unk_id != -1) { LLAMA_LOG_INFO( "%s: UNK token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].text.c_str() ); }+ if (vocab.special_sep_id != -1) { LLAMA_LOG_INFO( "%s: SEP token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].text.c_str() ); }+ if (vocab.special_pad_id != -1) { LLAMA_LOG_INFO( "%s: PAD token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].text.c_str() ); }+ if (vocab.linefeed_id != -1) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }}static void llm_load_tensors(@@ -2444,7 +2868,7 @@ static void llm_load_tensors(ml.calc_sizes(ctx_size, mmapped_size);- LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MB\n", __func__, ctx_size/1024.0/1024.0);+ LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, ctx_size/1024.0/1024.0);// create the ggml context{@@ -2505,14 +2929,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU-#ifndef _WIN32- backend_norm = llama_backend_offload;-#else- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;-#endif // _WIN32-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -2550,16 +2967,33 @@ static void llm_load_tensors(layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);// optional bias tensors- layer.bq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, backend, true);- layer.bk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, backend, true);- layer.bv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, backend, true);- layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend, true);+ layer.bq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, backend, false);+ layer.bk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, backend, false);+ layer.bv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, backend, false);+ layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend, false);layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);- layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_gate_inp = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd}, backend, false);++ if (layer.ffn_gate_inp == nullptr) {+ GGML_ASSERT(hparams.n_expert == 0);+ GGML_ASSERT(hparams.n_expert_used == 0);++ layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);+ layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ } else {+ GGML_ASSERT(hparams.n_expert > 0);+ GGML_ASSERT(hparams.n_expert_used > 0);++ // MoE branch+ for (uint32_t x = 0; x < hparams.n_expert; ++x) {+ layer.ffn_gate_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), {n_embd, n_ff}, backend_split);+ layer.ffn_down_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd}, backend_split);+ layer.ffn_up_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), {n_embd, n_ff}, backend_split);+ }+ }if (backend == GGML_BACKEND_GPU) {vram_weights +=@@ -2569,8 +3003,18 @@ static void llm_load_tensors((layer.bk ? ggml_nbytes(layer.bk) : 0) +(layer.bv ? ggml_nbytes(layer.bv) : 0) +(layer.bo ? ggml_nbytes(layer.bo) : 0) +- ggml_nbytes(layer.ffn_norm) + ggml_nbytes(layer.ffn_gate) +- ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);+ ggml_nbytes(layer.ffn_norm);++ if (layer.ffn_gate_inp == nullptr) {+ vram_weights +=+ ggml_nbytes(layer.ffn_gate) + ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);+ } else {+ vram_weights += ggml_nbytes(layer.ffn_gate_inp);+ for (uint32_t x = 0; x < hparams.n_expert; ++x) {+ vram_weights +=+ ggml_nbytes(layer.ffn_gate_exp[x]) + ggml_nbytes(layer.ffn_down_exp[x]) + ggml_nbytes(layer.ffn_up_exp[x]);+ }+ }}}} break;@@ -2582,14 +3026,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU-#ifndef _WIN32- backend_norm = llama_backend_offload;-#else- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;-#endif // _WIN32-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -2652,14 +3089,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU-#ifndef _WIN32- backend_norm = llama_backend_offload;-#else- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;-#endif // _WIN32-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -2729,14 +3159,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU-#ifndef _WIN32- backend_norm = llama_backend_offload;-#else- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;-#endif // _WIN32-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -2806,14 +3229,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU-#ifndef _WIN32- backend_norm = llama_backend_offload;-#else- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;-#endif // _WIN32-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -2872,14 +3288,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU-#ifndef _WIN32- backend_norm = llama_backend_offload;-#else- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;-#endif // _WIN32-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -2950,14 +3359,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU-#ifndef _WIN32- backend_norm = llama_backend_offload;-#else- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;-#endif // _WIN32-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -3017,14 +3419,7 @@ static void llm_load_tensors(ggml_backend_type backend_output;if (n_gpu_layers > int(n_layer)) {- // norm is not performance relevant on its own but keeping it in VRAM reduces data copying- // on Windows however this is detrimental unless everything is on the GPU- if (!IsWindows()) {- backend_norm = llama_backend_offload;- } else {- backend_norm = n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : llama_backend_offload;- }-+ backend_norm = llama_backend_offload;backend_output = llama_backend_offload_split;} else {backend_norm = GGML_BACKEND_CPU;@@ -3081,6 +3476,64 @@ static void llm_load_tensors(}}} break;+ case LLM_ARCH_QWEN:+ {+ model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ {+ ggml_backend_type backend_norm;+ ggml_backend_type backend_output;++ if (n_gpu_layers > int(n_layer)) {+ backend_norm = llama_backend_offload;+ backend_output = llama_backend_offload_split;+ } else {+ backend_norm = GGML_BACKEND_CPU;+ backend_output = GGML_BACKEND_CPU;+ }++ model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);+ model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);++ if (backend_norm == GGML_BACKEND_GPU) {+ vram_weights += ggml_nbytes(model.output_norm);+ }+ if (backend_output == GGML_BACKEND_GPU_SPLIT) {+ vram_weights += ggml_nbytes(model.output);+ }+ }++ const uint32_t n_ff = hparams.n_ff / 2;++ const int i_gpu_start = n_layer - n_gpu_layers;++ model.layers.resize(n_layer);++ for (uint32_t i = 0; i < n_layer; ++i) {+ const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT+ const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);++ layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd * 3}, backend_split);+ layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd * 3}, backend);+ layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);++ layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);++ layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);+ layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);++ if (backend == GGML_BACKEND_GPU) {+ vram_weights +=+ ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.wqkv) + ggml_nbytes(layer.bqkv) ++ ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) + ggml_nbytes(layer.ffn_gate) ++ ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);+ }+ }+ } break;default:ThrowRuntimeError("unknown architecture");@@ -3096,8 +3549,9 @@ static void llm_load_tensors(ctx_size +mmapped_size - vram_weights; // weights in VRAM not in memory- LLAMA_LOG_INFO("%s: mem required = %7.2f MB\n", __func__, mem_required / 1024.0 / 1024.0);+ LLAMA_LOG_INFO("%s: mem required = %7.2f MiB\n", __func__, mem_required / 1024.0 / 1024.0);+ if (!ggml_metal_supported() && ggml_cublas_loaded()) {const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer));LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_gpu);@@ -3105,11 +3559,12 @@ static void llm_load_tensors(LLAMA_LOG_INFO("%s: offloading non-repeating layers to GPU\n", __func__);}- const int max_backend_supported_layers = hparams.n_layer + 3;- const int max_offloadable_layers = hparams.n_layer + 3;+ const int max_backend_supported_layers = hparams.n_layer + 1;+ const int max_offloadable_layers = hparams.n_layer + 1;LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);- LLAMA_LOG_INFO("%s: VRAM used: %.2f MB\n", __func__, vram_weights / 1024.0 / 1024.0);+ LLAMA_LOG_INFO("%s: VRAM used: %.2f MiB\n", __func__, vram_weights / 1024.0 / 1024.0);+ }}// populate `tensors_by_name`@@ -3119,6 +3574,7 @@ static void llm_load_tensors(}(void) tensor_split;+ if (!ggml_metal_supported() && ggml_cublas_loaded()){ggml_cuda_set_tensor_split(tensor_split);}@@ -3140,7 +3596,7 @@ static bool llama_model_load(const std::string & fname, llama_model & model, con#ifndef _LIBCPP_NO_EXCEPTIONStry {#endif- llama_model_loader ml(fname, params.use_mmap);+ llama_model_loader ml(fname, params.use_mmap, params.kv_overrides);model.hparams.vocab_only = params.vocab_only;@@ -3269,11 +3725,11 @@ static void llm_build_k_shift(struct ggml_tensor * tmp =// we rotate only the first n_rot dimensionsggml_rope_custom_inplace(ctx,- ggml_view_3d(ctx, kv.k,+ ggml_view_3d(ctx, kv.k_l[il],n_embd_head, n_head_kv, n_ctx,- ggml_element_size(kv.k)*n_embd_head,- ggml_element_size(kv.k)*n_embd_gqa,- ggml_element_size(kv.k)*n_embd_gqa*n_ctx*il),+ ggml_type_sizef(kv.k_l[il]->type)*n_embd_head,+ ggml_type_sizef(kv.k_l[il]->type)*n_embd_gqa,+ 0),K_shift, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(tmp, "K_shifted", il);@@ -3300,13 +3756,13 @@ static void llm_build_kv_store(//struct ggml_tensor * v_cur_t = ggml_transpose(ctx, v_cur); // TODO: reshape above is likely not neededcb(v_cur_t, "v_cur_t", il);- struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k, n_tokens*n_embd_gqa,- (ggml_element_size(kv.k)*n_embd_gqa)*(il*n_ctx + kv_head));+ struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k_l[il], n_tokens*n_embd_gqa,+ (ggml_type_sizef(kv.k_l[il]->type)*n_embd_gqa)*kv_head);cb(k_cache_view, "k_cache_view", il);- struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, kv.v, n_tokens, n_embd_gqa,- ( n_ctx)*ggml_element_size(kv.v),- (il*n_ctx)*ggml_element_size(kv.v)*n_embd_gqa + kv_head*ggml_element_size(kv.v));+ struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_gqa,+ ( n_ctx)*ggml_element_size(kv.v_l[il]),+ (kv_head)*ggml_element_size(kv.v_l[il]));cb(v_cache_view, "v_cache_view", il);// important: storing RoPE-ed version of K in the KV cache!@@ -3458,40 +3914,46 @@ static struct ggml_tensor * llm_build_kqv(cb(q, "q", il);struct ggml_tensor * k =- ggml_view_3d(ctx, kv.k,+ ggml_view_3d(ctx, kv.k_l[il],n_embd_head, n_kv, n_head_kv,- ggml_element_size(kv.k)*n_embd_gqa,- ggml_element_size(kv.k)*n_embd_head,- ggml_element_size(kv.k)*n_embd_gqa*n_ctx*il);+ ggml_type_sizef(kv.k_l[il]->type)*n_embd_gqa,+ ggml_type_sizef(kv.k_l[il]->type)*n_embd_head,+ 0);cb(k, "k", il);struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);cb(kq, "kq", il);- kq = ggml_scale(ctx, kq, kq_scale);- cb(kq, "kq_scaled", il);-if (max_alibi_bias > 0.0f) {- // TODO: n_head or n_head_kv- // TODO: K-shift is likely not working- // TODO: change to ggml_add- kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, max_alibi_bias);- cb(kq, "kq_scaled_alibi", il);- }+ // temporary branch until we figure out how to handle ggml_alibi through ggml_add+ kq = ggml_scale(ctx, kq, kq_scale);+ cb(kq, "kq_scaled", il);++ if (max_alibi_bias > 0.0f) {+ // TODO: n_head or n_head_kv+ // TODO: K-shift is likely not working+ // TODO: change to ggml_add+ kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, max_alibi_bias);+ cb(kq, "kq_scaled_alibi", il);+ }- kq = ggml_add(ctx, kq, kq_mask);- cb(kq, "kq_masked", il);+ kq = ggml_add(ctx, kq, kq_mask);+ cb(kq, "kq_masked", il);- kq = ggml_soft_max(ctx, kq);- cb(kq, "kq_soft_max", il);+ kq = ggml_soft_max(ctx, kq);+ cb(kq, "kq_soft_max", il);+ } else {+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, 1.0f/sqrtf(float(n_embd_head)));+ cb(kq, "kq_soft_max_ext", il);+ }// split cached v into n_head headsstruct ggml_tensor * v =- ggml_view_3d(ctx, kv.v,+ ggml_view_3d(ctx, kv.v_l[il],n_kv, n_embd_head, n_head_kv,- ggml_element_size(kv.v)*n_ctx,- ggml_element_size(kv.v)*n_ctx*n_embd_head,- ggml_element_size(kv.v)*n_ctx*n_embd_gqa*il);+ ggml_element_size(kv.v_l[il])*n_ctx,+ ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head,+ 0);cb(v, "v", il);struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq);@@ -3529,6 +3991,8 @@ struct llm_build_context {const int64_t n_head_kv;const int64_t n_embd_head;const int64_t n_embd_gqa;+ const int64_t n_expert;+ const int64_t n_expert_used;const float freq_base;const float freq_scale;@@ -3570,6 +4034,8 @@ struct llm_build_context {n_head_kv (hparams.n_head_kv),n_embd_head (hparams.n_embd_head()),n_embd_gqa (hparams.n_embd_gqa()),+ n_expert (hparams.n_expert),+ n_expert_used (hparams.n_expert_used),freq_base (cparams.rope_freq_base),freq_scale (cparams.rope_freq_scale),ext_factor (cparams.yarn_ext_factor),@@ -3608,7 +4074,7 @@ struct llm_build_context {}struct ggml_cgraph * build_llama() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);GGML_ASSERT(n_embd_head == hparams.n_rot);@@ -3694,7 +4160,7 @@ struct llm_build_context {cb(ffn_inp, "ffn_inp", il);// feed-forward network- {+ if (model.layers[il].ffn_gate_inp == nullptr) {cur = llm_build_norm(ctx0, ffn_inp, hparams,model.layers[il].ffn_norm, NULL,LLM_NORM_RMS, cb, il);@@ -3706,6 +4172,69 @@ struct llm_build_context {model.layers[il].ffn_down, NULL,LLM_FFN_SILU, LLM_FFN_PAR, cb, il);cb(cur, "ffn_out", il);+ } else {+ // MoE branch+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ ggml_tensor * logits = ggml_mul_mat(ctx0, model.layers[il].ffn_gate_inp, cur); // [n_tokens, num_experts]+ cb(logits, "ffn_moe_logits", il);++ ggml_tensor * probs = ggml_soft_max(ctx0, logits); // [n_tokens, num_experts]+ cb(probs, "ffn_moe_probs", il);++ // select experts+ ggml_tensor * selected_experts = ggml_top_k(ctx0, probs, n_expert_used); // [n_tokens, num_experts_per_tok]+ cb(selected_experts->src[0], "ffn_moe_argsort", il);++ ggml_tensor * weights = ggml_get_rows(ctx0,+ ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected_experts);+ cb(weights, "ffn_moe_weights", il);++ weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens); // [n_tokens, num_experts_per_tok]++ ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights);+ cb(weights_sum, "ffn_moe_weights_sum", il);++ weights = ggml_div(ctx0, weights, weights_sum); // [n_tokens, num_experts_per_tok]+ cb(weights, "ffn_moe_weights_norm", il);++ // compute expert outputs+ ggml_tensor * moe_out = nullptr;++ for (int i = 0; i < n_expert_used; ++i) {+ ggml_tensor * cur_expert;++ ggml_tensor * cur_up = ggml_mul_mat_id(ctx0, model.layers[il].ffn_up_exp, n_expert, selected_experts, i, cur);+ cb(cur_up, "ffn_moe_up", il);++ ggml_tensor * cur_gate = ggml_mul_mat_id(ctx0, model.layers[il].ffn_gate_exp, n_expert, selected_experts, i, cur);+ cb(cur_gate, "ffn_moe_gate", il);++ cur_gate = ggml_silu(ctx0, cur_gate);+ cb(cur_gate, "ffn_moe_silu", il);++ cur_expert = ggml_mul(ctx0, cur_up, cur_gate); // [n_tokens, n_embd]+ cb(cur_expert, "ffn_moe_gate_par", il);++ cur_expert = ggml_mul_mat_id(ctx0, model.layers[il].ffn_down_exp, n_expert, selected_experts, i, cur_expert); // [n_tokens, n_embd]+ cb(cur_expert, "ffn_moe_down", il);++ cur_expert = ggml_mul(ctx0, cur_expert,+ ggml_view_2d(ctx0, weights, 1, n_tokens, weights->nb[1], i*weights->nb[0]));+ cb(cur_expert, "ffn_moe_weighted", il);++ if (i == 0) {+ moe_out = cur_expert;+ } else {+ moe_out = ggml_add(ctx0, moe_out, cur_expert);+ cb(moe_out, "ffn_moe_out", il);+ }+ }++ cur = moe_out;}cur = ggml_add(ctx0, cur, ffn_inp);@@ -3732,7 +4261,7 @@ struct llm_build_context {}struct ggml_cgraph * build_baichuan() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -3852,7 +4381,7 @@ struct llm_build_context {}struct ggml_cgraph * build_falcon() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -3974,7 +4503,7 @@ struct llm_build_context {}struct ggml_cgraph * build_starcoder() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);struct ggml_tensor * cur;struct ggml_tensor * pos;@@ -4073,7 +4602,7 @@ struct llm_build_context {}struct ggml_cgraph * build_persimmon() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);const int64_t n_rot = n_embd_head / 2;@@ -4083,6 +4612,7 @@ struct llm_build_context {inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);cb(inpL, "imp_embd", -1);+ // inp_pos - contains the positionsstruct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);@@ -4090,6 +4620,7 @@ struct llm_build_context {struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);cb(KQ_scale, "KQ_scale", -1);+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -4218,7 +4749,7 @@ struct llm_build_context {struct ggml_tensor * Kcur = ggml_concat(ctx0, krotated, kpass);cb(Kcur, "Kcur", il);- struct ggml_tensor * Q = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 1, 2, 0, 3));+ struct ggml_tensor * Q = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 2, 1, 0, 3));cb(Q, "Q", il);Kcur = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 2, 1, 0, 3));@@ -4283,7 +4814,7 @@ struct llm_build_context {}struct ggml_cgraph * build_refact() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -4374,7 +4905,7 @@ struct llm_build_context {}struct ggml_cgraph * build_bloom() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -4468,7 +4999,7 @@ struct llm_build_context {}struct ggml_cgraph * build_mpt() {- struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -4678,6 +5209,121 @@ struct llm_build_context {return gf;}++ struct ggml_cgraph * build_qwen() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ cb(inpL, "inp_embd", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos= ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ cb(inp_pos, "inp_pos", -1);++ // KQ_scale+ struct ggml_tensor * KQ_scale= ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);+ cb(KQ_scale, "KQ_scale", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask= ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;++ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur);+ cb(cur, "wqkv", il);++ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);+ cb(cur, "bqkv", il);++ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*(n_embd)));++ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);++ // using mode = 2 for neox mode+ Qcur = ggml_rope_custom(+ ctx0, Qcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx,+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, Kcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx,+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);++ cur = llm_build_kqv(ctx0, hparams, kv_self,+ model.layers[il].wo, NULL,+ Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, cb, il);+ cb(cur, "kqv_out", il);+ }++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward forward+ {+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);+ }++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }};//@@ -4688,8 +5334,8 @@ struct llm_build_context {enum llm_offload_func_e {OFFLOAD_FUNC_NOP,OFFLOAD_FUNC,- OFFLOAD_FUNC_KQ,- OFFLOAD_FUNC_V,+ OFFLOAD_FUNC_FRC, // force offload+ OFFLOAD_FUNC_KQV,OFFLOAD_FUNC_NR,OFFLOAD_FUNC_EMB,OFFLOAD_FUNC_OUT,@@ -4775,11 +5421,12 @@ static const std::unordered_map<const char *, llm_offload_func_e> k_offload_map//{ "inp_embd", OFFLOAD_FUNC_NR }, // TODO: missing K-quants get_rows kernel{ "pos_embd", OFFLOAD_FUNC_NR },- { "inp_pos", OFFLOAD_FUNC_KQ }, // this is often used for KQ ops (e.g. rope)- { "KQ_scale", OFFLOAD_FUNC_KQ },- { "KQ_mask", OFFLOAD_FUNC_KQ },- { "K_shift", OFFLOAD_FUNC_KQ },- { "K_shifted", OFFLOAD_FUNC_KQ },+ { "inp_pos", OFFLOAD_FUNC_FRC }, // this is often used for KQ ops (e.g. rope)+ { "KQ_scale", OFFLOAD_FUNC_FRC },+ { "KQ_mask", OFFLOAD_FUNC_FRC },+ { "K_shift", OFFLOAD_FUNC_FRC },++ { "K_shifted", OFFLOAD_FUNC },{ "inp_norm", OFFLOAD_FUNC_NR },{ "inp_norm_w", OFFLOAD_FUNC_NR },@@ -4792,37 +5439,38 @@ static const std::unordered_map<const char *, llm_offload_func_e> k_offload_map{ "attn_norm", OFFLOAD_FUNC },{ "attn_norm_2", OFFLOAD_FUNC },- { "wqkv", OFFLOAD_FUNC_KQ },- { "bqkv", OFFLOAD_FUNC_KQ },- { "wqkv_clamped", OFFLOAD_FUNC_KQ },-- { "tmpk", OFFLOAD_FUNC_KQ },- { "tmpq", OFFLOAD_FUNC_KQ },- { "tmpv", OFFLOAD_FUNC_V },- { "Kcur", OFFLOAD_FUNC_KQ },- { "Qcur", OFFLOAD_FUNC_KQ },- { "Vcur", OFFLOAD_FUNC_V },-- { "krot", OFFLOAD_FUNC_KQ },- { "qrot", OFFLOAD_FUNC_KQ },- { "kpass", OFFLOAD_FUNC_KQ },- { "qpass", OFFLOAD_FUNC_KQ },- { "krotated", OFFLOAD_FUNC_KQ },- { "qrotated", OFFLOAD_FUNC_KQ },-- { "q", OFFLOAD_FUNC_KQ },- { "k", OFFLOAD_FUNC_KQ },- { "kq", OFFLOAD_FUNC_KQ },- { "kq_scaled", OFFLOAD_FUNC_KQ },- { "kq_scaled_alibi", OFFLOAD_FUNC_KQ },- { "kq_masked", OFFLOAD_FUNC_KQ },- { "kq_soft_max", OFFLOAD_FUNC_V },- { "v", OFFLOAD_FUNC_V },- { "kqv", OFFLOAD_FUNC_V },- { "kqv_merged", OFFLOAD_FUNC_V },- { "kqv_merged_cont", OFFLOAD_FUNC_V },- { "kqv_wo", OFFLOAD_FUNC_V },- { "kqv_out", OFFLOAD_FUNC_V },+ { "wqkv", OFFLOAD_FUNC_KQV },+ { "bqkv", OFFLOAD_FUNC_KQV },+ { "wqkv_clamped", OFFLOAD_FUNC_KQV },++ { "tmpk", OFFLOAD_FUNC_KQV },+ { "tmpq", OFFLOAD_FUNC_KQV },+ { "tmpv", OFFLOAD_FUNC_KQV },+ { "Kcur", OFFLOAD_FUNC_KQV },+ { "Qcur", OFFLOAD_FUNC_KQV },+ { "Vcur", OFFLOAD_FUNC_KQV },++ { "krot", OFFLOAD_FUNC_KQV },+ { "qrot", OFFLOAD_FUNC_KQV },+ { "kpass", OFFLOAD_FUNC_KQV },+ { "qpass", OFFLOAD_FUNC_KQV },+ { "krotated", OFFLOAD_FUNC_KQV },+ { "qrotated", OFFLOAD_FUNC_KQV },++ { "q", OFFLOAD_FUNC_KQV },+ { "k", OFFLOAD_FUNC_KQV },+ { "kq", OFFLOAD_FUNC_KQV },+ { "kq_scaled", OFFLOAD_FUNC_KQV },+ { "kq_scaled_alibi", OFFLOAD_FUNC_KQV },+ { "kq_masked", OFFLOAD_FUNC_KQV },+ { "kq_soft_max", OFFLOAD_FUNC_KQV },+ { "kq_soft_max_ext", OFFLOAD_FUNC_KQV },+ { "v", OFFLOAD_FUNC_KQV },+ { "kqv", OFFLOAD_FUNC_KQV },+ { "kqv_merged", OFFLOAD_FUNC_KQV },+ { "kqv_merged_cont", OFFLOAD_FUNC_KQV },+ { "kqv_wo", OFFLOAD_FUNC_KQV },+ { "kqv_out", OFFLOAD_FUNC_KQV },{ "ffn_inp", OFFLOAD_FUNC },{ "ffn_norm", OFFLOAD_FUNC },@@ -4841,6 +5489,20 @@ static const std::unordered_map<const char *, llm_offload_func_e> k_offload_map{ "ffn_relu", OFFLOAD_FUNC },{ "ffn_sqr(relu)", OFFLOAD_FUNC },+ { "ffn_moe_logits", OFFLOAD_FUNC },+ { "ffn_moe_probs", OFFLOAD_FUNC },+ { "ffn_moe_argsort", OFFLOAD_FUNC },+ { "ffn_moe_weights", OFFLOAD_FUNC },+ { "ffn_moe_weights_sum", OFFLOAD_FUNC },+ { "ffn_moe_weights_norm", OFFLOAD_FUNC },+ { "ffn_moe_weighted", OFFLOAD_FUNC },+ { "ffn_moe_up", OFFLOAD_FUNC },+ { "ffn_moe_gate", OFFLOAD_FUNC },+ { "ffn_moe_silu", OFFLOAD_FUNC },+ { "ffn_moe_gate_par", OFFLOAD_FUNC },+ { "ffn_moe_down", OFFLOAD_FUNC },+ { "ffn_moe_out", OFFLOAD_FUNC },+{ "l_out", OFFLOAD_FUNC },{ "result_norm", OFFLOAD_FUNC_EMB },@@ -5009,10 +5671,10 @@ static struct ggml_cgraph * llama_build_graph(static const std::unordered_map<llm_offload_func_e, std::string, std::hash<int>> k_offload_func_name = {{ OFFLOAD_FUNC_NOP, "CPU" },{ OFFLOAD_FUNC_OUT, "CPU" },- { OFFLOAD_FUNC, "GPU (CUDA)" },- { OFFLOAD_FUNC_KQ, "GPU (CUDA) KQ" },- { OFFLOAD_FUNC_V, "GPU (CUDA) V" },- { OFFLOAD_FUNC_NR, "GPU (CUDA) NR" },+ { OFFLOAD_FUNC, "GPU (CUDA)" },+ { OFFLOAD_FUNC_FRC, "GPU (CUDA) FRC" },+ { OFFLOAD_FUNC_KQV, "GPU (CUDA) KQV" },+ { OFFLOAD_FUNC_NR, "GPU (CUDA) NR" },{ OFFLOAD_FUNC_EMB, "GPU (CUDA) EMB" },};@@ -5043,18 +5705,23 @@ static struct ggml_cgraph * llama_build_graph(}}break;- case OFFLOAD_FUNC_NR:- if (n_gpu_layers <= n_layer + 0) {+ case OFFLOAD_FUNC_FRC:+ if (!lctx.cparams.offload_kqv) {func_e = OFFLOAD_FUNC_NOP;- }- break;- case OFFLOAD_FUNC_V:- if (n_gpu_layers <= n_layer + 1) {+ } break;+ case OFFLOAD_FUNC_KQV:+ if (!lctx.cparams.offload_kqv) {func_e = OFFLOAD_FUNC_NOP;+ } else {+ if (n_gpu_layers < n_layer) {+ if (il < i_gpu_start) {+ func_e = OFFLOAD_FUNC_NOP;+ }+ }}break;- case OFFLOAD_FUNC_KQ:- if (n_gpu_layers <= n_layer + 2) {+ case OFFLOAD_FUNC_NR:+ if (n_gpu_layers <= n_layer + 0) {func_e = OFFLOAD_FUNC_NOP;}break;@@ -5069,14 +5736,19 @@ static struct ggml_cgraph * llama_build_graph(offload_func_t func = ggml_offload_nop;// this is needed for compatibility with Metal for example- static offload_func_t ggml_offload_gpu = ggml_cuda_assign_buffers_no_alloc;+ static offload_func_t ggml_offload_gpu;+ if (!ggml_metal_supported() && ggml_cublas_loaded()) {+ ggml_offload_gpu = ggml_cuda_assign_buffers_no_alloc;+ } else {+ ggml_offload_gpu = ggml_offload_nop;+ }switch (func_e) {case OFFLOAD_FUNC_NOP:case OFFLOAD_FUNC_OUT: func = ggml_offload_nop; break;case OFFLOAD_FUNC:- case OFFLOAD_FUNC_KQ:- case OFFLOAD_FUNC_V:+ case OFFLOAD_FUNC_KQV:+ case OFFLOAD_FUNC_FRC:case OFFLOAD_FUNC_NR:case OFFLOAD_FUNC_EMB: func = ggml_offload_gpu; break;default: GGML_ASSERT(false);@@ -5135,6 +5807,10 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_stablelm();} break;+ case LLM_ARCH_QWEN:+ {+ result = llm.build_qwen();+ } break;default:GGML_ASSERT(false);}@@ -5244,6 +5920,12 @@ static int llama_decode_internal(batch.seq_id = seq_id_arr.data();}+ // if we have enough unused cells before the current head ->+ // better to start searching from the beginning of the cache, hoping to fill it+ if (kv_self.head > kv_self.used + 2*n_tokens) {+ kv_self.head = 0;+ }+if (!llama_kv_cache_find_slot(kv_self, batch)) {return 1;}@@ -5251,10 +5933,10 @@ static int llama_decode_internal(// a heuristic, to avoid attending the full cache if it is not yet utilized// after enough generations, the benefit from this heuristic disappears// if we start defragmenting the cache, the benefit from this will be more important- //kv_self.n = std::max(32, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32)); // TODO: this might be better for CUDA?- kv_self.n = std::min((int32_t) cparams.n_ctx, std::max(32, llama_kv_cache_cell_max(kv_self)));+ kv_self.n = std::min((int32_t) cparams.n_ctx, std::max(32, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32)));+ //kv_self.n = llama_kv_cache_cell_max(kv_self);- //printf("kv_self.n = %d\n", kv_self.n);+ //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head);ggml_allocr_reset(lctx.alloc);@@ -5269,7 +5951,7 @@ static int llama_decode_internal(GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0);- if (!ggml_metal_supported() && ggml_cuda_supported()) {+ if (!ggml_metal_supported() && ggml_cublas_loaded()) {for (int i = 0; i < gf->n_leafs; i++) {ggml_tensor * node = gf->leafs[i];if (node->backend == GGML_BACKEND_GPU && node->extra == NULL) {@@ -5303,18 +5985,8 @@ static int llama_decode_internal(n_threads = std::min(4, n_threads);}- // If all tensors can be run on the GPU then using more than 1 thread is detrimental.- const bool full_offload_supported =- model.arch == LLM_ARCH_LLAMA ||- model.arch == LLM_ARCH_BAICHUAN ||- model.arch == LLM_ARCH_FALCON ||- model.arch == LLM_ARCH_REFACT ||- model.arch == LLM_ARCH_MPT ||- model.arch == LLM_ARCH_STARCODER ||- model.arch == LLM_ARCH_STABLELM;-- const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 3;- if (ggml_cpu_has_cublas() && full_offload_supported && fully_offloaded) {+ const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 1;+ if (ggml_cpu_has_cublas() && fully_offloaded) {n_threads = 1;}@@ -6099,7 +6771,10 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &// by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer// and passing 'add space prefix' as bool argument//- auto raw_text = (special ? "" : " ") + fragment.raw_text.substr(fragment.offset, fragment.length);+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);+ if (&fragment == &fragment_buffer.front()) {+ raw_text = " " + raw_text; // prefix with space if the first token is not special+ }#ifdef PRETOKENIZERDEBUGfprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());@@ -6165,14 +6840,13 @@ struct llama_grammar_candidate {// Decodes a UTF-8 string which may end in an incomplete sequence. Adds a terminating 0 for use as// pointer. If an invalid sequence is encountered, returns `llama_partial_utf8.n_remain == -1`.static std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(- const char * src,- size_t n_src,+ const std::string & src,llama_partial_utf8 partial_start) {static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 };- const char * pos = src;+ const char * pos = src.c_str();std::vector<uint32_t> code_points;// common english strings have the same number of codepoints and bytes. `+ 1` for the terminating 0.- code_points.reserve(n_src + 1);+ code_points.reserve(src.size() + 1);uint32_t value = partial_start.value;int n_remain = partial_start.n_remain;@@ -6223,13 +6897,6 @@ static std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(return std::make_pair(std::move(code_points), llama_partial_utf8{ value, n_remain });}-static std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(- std::string src,- llama_partial_utf8 partial_start-) {- return decode_utf8(src.c_str(), src.size(), partial_start);-}-// returns true iff pos points to the end of one of the definitions of a rulestatic bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) {switch (pos->type) {@@ -6868,11 +7535,13 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * cconst llama_token eos = llama_token_eos(&ctx->model);std::vector<std::pair<std::vector<uint32_t>, llama_partial_utf8>> candidates_decoded;+ candidates_decoded.reserve(candidates->size);std::vector<llama_grammar_candidate> candidates_grammar;+ candidates_grammar.reserve(candidates->size);for (size_t i = 0; i < candidates->size; ++i) {const llama_token id = candidates->data[i].id;- const std::string piece = llama_token_to_piece(ctx, id);+ const std::string & piece = ctx->model.vocab.id_to_token[id].text;if (id == eos) {if (!allow_eos) {candidates->data[i].logit = -INFINITY;@@ -7084,7 +7753,7 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammarGGML_ASSERT(false);}- const std::string piece = llama_token_to_piece(ctx, token);+ const std::string & piece = ctx->model.vocab.id_to_token[token].text;// Note terminating 0 in decoded stringconst auto decoded = decode_utf8(piece, grammar->partial_utf8);@@ -7436,11 +8105,9 @@ static void llama_convert_tensor_internal(workers.clear();}-static ggml_type get_k_quant_type(- quantize_state_internal & qs,- ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype-) {+static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {const std::string name = ggml_get_name(tensor);+// TODO: avoid hardcoded tensor names - use the TN_* constantsconst llm_arch arch = qs.model.arch;const auto tn = LLM_TN(arch);@@ -7474,7 +8141,18 @@ static ggml_type get_k_quant_type(// nearly negligible increase in model size by quantizing this tensor with more bits:if (new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K) new_type = GGML_TYPE_Q5_K;}+ if (qs.model.hparams.n_expert == 8) {+ // for the 8-expert model, bumping this to Q8_0 trades just ~128MB+ // TODO: explore better strategies+ new_type = GGML_TYPE_Q8_0;+ }++qs.i_attention_wv;+ } else if (name.find("attn_k.weight") != std::string::npos) {+ if (qs.model.hparams.n_expert == 8) {+ // for the 8-expert model, bumping this to Q8_0 trades just ~128MB+ // TODO: explore better strategies+ new_type = GGML_TYPE_Q8_0;+ }} else if (name.find("ffn_down.weight") != std::string::npos) {if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {@@ -7585,7 +8263,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s// hot cache. It may cause a slowdown on macOS, possibly related to free memory.bool use_mmap = !IsXnu();- llama_model_loader ml(fname_inp, use_mmap);+ llama_model_loader ml(fname_inp, use_mmap, NULL);if (ml.use_mmap) {ml.mapping.reset(new llama_mmap(&ml.file, /* prefetch */ 0, ggml_is_numa()));}@@ -7683,6 +8361,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const squantize &= params->quantize_output_tensor || name != "output.weight";quantize &= !params->only_copy;+ // do not quantize expert gating tensors+ quantize &= name.find("ffn_gate_inp.weight") == std::string::npos;+enum ggml_type new_type;void * new_data;size_t new_size;@@ -7761,7 +8442,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sworkers.clear();}- LLAMA_LOG_INFO("size = %8.2f MB -> %8.2f MB | hist: ", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0);+ LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB | hist: ", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0);int64_t tot_count = 0;for (size_t i = 0; i < hist_cur.size(); i++) {hist_all[i] += hist_cur[i];@@ -7881,7 +8562,7 @@ static int llama_apply_lora_from_file_internal(std::vector<uint8_t> base_buf;if (path_base_model) {LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model);- ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true));+ ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ NULL));size_t ctx_size;size_t mmapped_size;@@ -7989,7 +8670,7 @@ static int llama_apply_lora_from_file_internal(offload_func_t offload_func = ggml_offload_nop;offload_func_t offload_func_force_inplace = ggml_offload_nop;- if (!ggml_metal_supported() && ggml_cuda_supported()) {+ if (!ggml_metal_supported() && ggml_cublas_loaded()) {if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) {if (dest_t->type != GGML_TYPE_F16) {ThrowRuntimeError(format(@@ -8109,6 +8790,7 @@ struct llama_model_params llama_model_default_params() {/*.tensor_split =*/ nullptr,/*.progress_callback =*/ nullptr,/*.progress_callback_user_data =*/ nullptr,+ /*.kv_overrides =*/ nullptr,/*.vocab_only =*/ false,/*.use_mmap =*/ true,/*.use_mlock =*/ false,@@ -8136,10 +8818,12 @@ struct llama_context_params llama_context_default_params() {/*.yarn_beta_fast =*/ 32.0f,/*.yarn_beta_slow =*/ 1.0f,/*.yarn_orig_ctx =*/ 0,+ /*.type_k =*/ GGML_TYPE_F16,+ /*.type_v =*/ GGML_TYPE_F16,/*.mul_mat_q =*/ true,- /*.f16_kv =*/ true,/*.logits_all =*/ false,/*.embedding =*/ false,+ /*.offload_kqv =*/ true,};return result;@@ -8256,6 +8940,7 @@ struct llama_context * llama_new_context_with_model(cparams.yarn_beta_fast = params.yarn_beta_fast;cparams.yarn_beta_slow = params.yarn_beta_slow;cparams.mul_mat_q = params.mul_mat_q;+ cparams.offload_kqv = params.offload_kqv;cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;@@ -8289,19 +8974,36 @@ struct llama_context * llama_new_context_with_model(ctx->rng = std::mt19937(params.seed);ctx->logits_all = params.logits_all;- ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32;+ const ggml_type type_k = params.type_k;+ const ggml_type type_v = params.type_v;++ GGML_ASSERT(hparams.n_embd_head() % ggml_blck_size(type_k) == 0);+ GGML_ASSERT(hparams.n_embd_head() % ggml_blck_size(type_v) == 0);// reserve memory for context buffersif (!hparams.vocab_only) {- if (!llama_kv_cache_init(ctx->model.hparams, ctx->kv_self, memory_type, cparams.n_ctx, model->n_gpu_layers)) {+ if (!llama_kv_cache_init(ctx->model.hparams, ctx->kv_self, type_k, type_v, cparams.n_ctx, model->n_gpu_layers, cparams.offload_kqv)) {LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__);llama_free(ctx);return nullptr;}{- const size_t memory_size = ggml_nbytes(ctx->kv_self.k) + ggml_nbytes(ctx->kv_self.v);- LLAMA_LOG_INFO("%s: kv self size = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);+ size_t memory_size_k = 0;+ size_t memory_size_v = 0;++ for (auto & k : ctx->kv_self.k_l) {+ memory_size_k += ggml_nbytes(k);+ }++ for (auto & v : ctx->kv_self.v_l) {+ memory_size_v += ggml_nbytes(v);+ }++ LLAMA_LOG_INFO("%s: KV self size = %7.2f MiB, K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__,+ (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),+ ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f),+ ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));}// resized during inference@@ -8318,7 +9020,7 @@ struct llama_context * llama_new_context_with_model({static const size_t tensor_alignment = 32;// the compute buffer is used to store the tensor and graph structs, while the allocator buffer is used for the tensor data- ctx->buf_compute.resize(ggml_tensor_overhead()*GGML_MAX_NODES + ggml_graph_overhead());+ ctx->buf_compute.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead());// create measure allocatorctx->alloc = ggml_allocr_new_measure(tensor_alignment);@@ -8329,8 +9031,9 @@ struct llama_context * llama_new_context_with_model(llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graphggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0));- if (ggml_metal_supported() && model->n_gpu_layers > 0) {- ctx->ctx_metal = ggml_metal_init(1, NULL);+ if (ggml_metal_supported()) {+ if (model->n_gpu_layers > 0) {+ ctx->ctx_metal = ggml_metal_init(1);if (!ctx->ctx_metal) {LLAMA_LOG_ERROR("%s: ggml_metal_init() failed\n", __func__);llama_free(ctx);@@ -8339,10 +9042,12 @@ struct llama_context * llama_new_context_with_model(//ggml_metal_graph_find_concurrency(ctx->ctx_metal, gf, false);//ggml_allocr_set_parse_seq(ctx->alloc, ggml_metal_get_concur_list(ctx->ctx_metal), ggml_metal_if_optimized(ctx->ctx_metal));}+ }+// measure memory requirements for the graphsize_t alloc_size = ggml_allocr_alloc_graph(ctx->alloc, gf) + tensor_alignment;- LLAMA_LOG_INFO("%s: compute buffer total size = %.2f MB\n", __func__, (ctx->buf_compute.size + alloc_size) / 1024.0 / 1024.0);+ LLAMA_LOG_INFO("%s: compute buffer total size = %.2f MiB\n", __func__, (ctx->buf_compute.size + alloc_size) / 1024.0 / 1024.0);// recreate allocator with exact memory requirementsggml_allocr_free(ctx->alloc);@@ -8354,7 +9059,7 @@ struct llama_context * llama_new_context_with_model(}if (!ggml_metal_supported() && ggml_cuda_supported()) {ggml_cuda_set_scratch_size(alloc_size);- LLAMA_LOG_INFO("%s: VRAM scratch buffer: %.2f MB\n", __func__, alloc_size / 1024.0 / 1024.0);+ LLAMA_LOG_INFO("%s: VRAM scratch buffer: %.2f MiB\n", __func__, alloc_size / 1024.0 / 1024.0);// calculate total VRAM usageauto add_tensor = [](const ggml_tensor * t, size_t & size) {@@ -8368,20 +9073,25 @@ struct llama_context * llama_new_context_with_model(}size_t kv_vram_size = 0;- add_tensor(ctx->kv_self.k, kv_vram_size);- add_tensor(ctx->kv_self.v, kv_vram_size);+ for (auto & k : ctx->kv_self.k_l) {+ add_tensor(k, kv_vram_size);+ }+ for (auto & v : ctx->kv_self.v_l) {+ add_tensor(v, kv_vram_size);+ }size_t ctx_vram_size = alloc_size + kv_vram_size;size_t total_vram_size = model_vram_size + ctx_vram_size;- LLAMA_LOG_INFO("%s: total VRAM used: %.2f MB (model: %.2f MB, context: %.2f MB)\n", __func__,+ LLAMA_LOG_INFO("%s: total VRAM used: %.2f MiB (model: %.2f MiB, context: %.2f MiB)\n", __func__,total_vram_size / 1024.0 / 1024.0,model_vram_size / 1024.0 / 1024.0,- ctx_vram_size / 1024.0 / 1024.0);+ ctx_vram_size / 1024.0 / 1024.0);}}- if (ggml_metal_supported() && model->n_gpu_layers > 0) {+ if (ggml_metal_supported()) {+ if (model->n_gpu_layers > 0) {// this allocates all Metal resources and memory buffersvoid * data_ptr = NULL;@@ -8397,7 +9107,7 @@ struct llama_context * llama_new_context_with_model(const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx);- LLAMA_LOG_INFO("%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0);+ LLAMA_LOG_INFO("%s: max tensor size = %8.2f MiB\n", __func__, max_size/1024.0/1024.0);#define LLAMA_METAL_CHECK_BUF(result) \if (!(result)) { \@@ -8411,6 +9121,7 @@ struct llama_context * llama_new_context_with_model(LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "alloc", ctx->buf_alloc.data, ctx->buf_alloc.size, 0));#undef LLAMA_METAL_CHECK_BUF}+ }}#ifdef GGML_USE_MPI@@ -8462,6 +9173,45 @@ float llama_rope_freq_scale_train(const struct llama_model * model) {return model->hparams.rope_freq_scale_train;}+int llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size) {+ const auto & it = model->gguf_kv.find(key);+ if (it == model->gguf_kv.end()) {+ if (buf_size > 0) {+ buf[0] = '\0';+ }+ return -1;+ }+ return snprintf(buf, buf_size, "%s", it->second.c_str());+}++int llama_model_meta_count(const struct llama_model * model) {+ return (int)model->gguf_kv.size();+}++int llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size) {+ if (i < 0 || i >= (int)model->gguf_kv.size()) {+ if (buf_size > 0) {+ buf[0] = '\0';+ }+ return -1;+ }+ auto it = model->gguf_kv.begin();+ std::advance(it, i);+ return snprintf(buf, buf_size, "%s", it->first.c_str());+}++int llama_model_meta_val_str_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size) {+ if (i < 0 || i >= (int)model->gguf_kv.size()) {+ if (buf_size > 0) {+ buf[0] = '\0';+ }+ return -1;+ }+ auto it = model->gguf_kv.begin();+ std::advance(it, i);+ return snprintf(buf, buf_size, "%s", it->second.c_str());+}+int llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {return snprintf(buf, buf_size, "%s %s %s",llama_model_arch_name(model->arch).c_str(),@@ -8532,8 +9282,107 @@ int llama_model_apply_lora_from_file(const struct llama_model * model, const cha#endif}+struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq) {+ struct llama_kv_cache_view result = {+ /*.n_cells = */ 0,+ /*.n_max_seq = */ n_max_seq,+ /*.token_count = */ 0,+ /*.used_cells = */ llama_get_kv_cache_used_cells(ctx),+ /*.max_contiguous = */ 0,+ /*.max_contiguous_idx = */ -1,+ /*.cells = */ nullptr,+ /*.cells_sequences = */ nullptr,+ };+ return result;+}++void llama_kv_cache_view_free(struct llama_kv_cache_view * view) {+ if (view->cells != nullptr) {+ free(view->cells);+ view->cells = nullptr;+ }+ if (view->cells_sequences != nullptr) {+ free(view->cells_sequences);+ view->cells_sequences = nullptr;+ }+}++void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view) {+ if (uint32_t(view->n_cells) < ctx->kv_self.size || view->cells == nullptr) {+ view->n_cells = int32_t(ctx->kv_self.size);+ void * p = realloc(view->cells, sizeof(struct llama_kv_cache_view_cell) * view->n_cells);+ GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells");+ view->cells = (struct llama_kv_cache_view_cell *)p;+ p = realloc(view->cells_sequences, sizeof(llama_seq_id) * view->n_max_seq * view->n_cells);+ GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells sequences");+ view->cells_sequences = (llama_seq_id *)p;+ }++ const std::vector<llama_kv_cell> & kv_cells = ctx->kv_self.cells;+ llama_kv_cache_view_cell * c_curr = view->cells;+ llama_seq_id * cs_curr = view->cells_sequences;+ int32_t used_cells = 0;+ int32_t token_count = 0;+ int32_t curr_contig_idx = -1;+ uint32_t max_contig = 0;+ int32_t max_contig_idx = -1;++ for (int32_t i = 0; i < int32_t(ctx->kv_self.size); i++, c_curr++, cs_curr += view->n_max_seq) {+ const size_t curr_size = kv_cells[i].seq_id.size();+ token_count += curr_size;+ c_curr->pos = kv_cells[i].pos + kv_cells[i].delta;++ if (curr_size > 0) {+ if (curr_contig_idx >= 0 && uint32_t(i - curr_contig_idx) > max_contig) {+ max_contig = i - curr_contig_idx;+ max_contig_idx = curr_contig_idx;+ }+ curr_contig_idx = -1;+ } else if (curr_contig_idx < 0) {+ curr_contig_idx = i;+ }++ int seq_idx = 0;+ for (const llama_seq_id it : kv_cells[i].seq_id) {+ if (seq_idx >= view->n_max_seq) {+ break;+ }+ cs_curr[seq_idx] = it;+ seq_idx++;+ }+ if (seq_idx != 0) {+ used_cells++;+ }+ for (; seq_idx < view->n_max_seq; seq_idx++) {+ cs_curr[seq_idx] = -1;+ }+ }+ if (curr_contig_idx >= 0 && kv_cells.size() - curr_contig_idx > max_contig) {+ max_contig_idx = curr_contig_idx;+ max_contig = kv_cells.size() - curr_contig_idx;+ }+ view->max_contiguous = max_contig;+ view->max_contiguous_idx = max_contig_idx;+ view->token_count = token_count;+ view->used_cells = used_cells;+ if (uint32_t(used_cells) != ctx->kv_self.used) {+ LLAMA_LOG_ERROR("%s: used cells mismatch. kv_cache says %d but we calculated %d\n",+ __func__, ctx->kv_self.used, used_cells);+ }+}+int llama_get_kv_cache_token_count(const struct llama_context * ctx) {- return ctx->kv_self.head;+ int result = 0;++ for (uint32_t i = 0; i < ctx->kv_self.size; i++) {+ result += ctx->kv_self.cells[i].seq_id.size();+ }++ return result;+}++int llama_get_kv_cache_used_cells(const struct llama_context * ctx) {+ return ctx->kv_self.used;}void llama_kv_cache_clear(struct llama_context * ctx) {@@ -8703,43 +9552,53 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_datconst size_t kv_buf_size = kv_self.buf.size;const uint32_t kv_head = kv_self.head;const uint32_t kv_size = kv_self.size;+ const uint32_t kv_used = kv_self.used;data_ctx->write(&kv_buf_size, sizeof(kv_buf_size));data_ctx->write(&kv_head, sizeof(kv_head));data_ctx->write(&kv_size, sizeof(kv_size));+ data_ctx->write(&kv_used, sizeof(kv_used));if (kv_buf_size) {- const size_t elt_size = ggml_element_size(kv_self.k);+ const size_t elt_size = ggml_element_size(kv_self.k_l[0]);++ ggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true });+ ggml_cgraph * gf = ggml_new_graph(cpy_ctx);++ std::vector<std::vector<uint8_t>> kout2d_data(n_layer);+ std::vector<std::vector<uint8_t>> vout2d_data(n_layer);- ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true });- ggml_cgraph gf{};+ for (int il = 0; il < (int) n_layer; ++il) {+ ggml_tensor * kout2d = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd, kv_head);+ kout2d_data[il].resize(ggml_nbytes(kout2d));+ kout2d->data = kout2d_data[il].data();- ggml_tensor * kout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_head, n_layer);- std::vector<uint8_t> kout3d_data(ggml_nbytes(kout3d), 0);- kout3d->data = kout3d_data.data();+ ggml_tensor * vout2d = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd);+ vout2d_data[il].resize(ggml_nbytes(vout2d));+ vout2d->data = vout2d_data[il].data();- ggml_tensor * vout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_head, n_embd, n_layer);- std::vector<uint8_t> vout3d_data(ggml_nbytes(vout3d), 0);- vout3d->data = vout3d_data.data();+ ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il],+ n_embd, kv_head,+ elt_size*n_embd, 0);- ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k,- n_embd, kv_head, n_layer,- elt_size*n_embd, elt_size*n_embd*n_ctx, 0);+ ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il],+ kv_head, n_embd,+ elt_size*n_ctx, 0);- ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v,- kv_head, n_embd, n_layer,- elt_size*n_ctx, elt_size*n_ctx*n_embd, 0);+ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, k2d, kout2d));+ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, v2d, vout2d));+ }- ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, k3d, kout3d));- ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, v3d, vout3d));- ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1);+ ggml_graph_compute_helper(ctx->work_buffer, gf, /*n_threads*/ 1);ggml_free(cpy_ctx);- // our data is now in the kout3d_data and vout3d_data buffers+ // our data is now in the kout2d_data and vout2d_data buffers// write them to file- data_ctx->write(kout3d_data.data(), kout3d_data.size());- data_ctx->write(vout3d_data.data(), vout3d_data.size());+ for (uint32_t il = 0; il < n_layer; ++il) {+ data_ctx->write(kout2d_data[il].data(), kout2d_data[il].size());+ data_ctx->write(vout2d_data[il].data(), vout2d_data[il].size());+ }}for (uint32_t i = 0; i < kv_size; ++i) {@@ -8829,44 +9688,50 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {size_t kv_buf_size;uint32_t kv_head;uint32_t kv_size;+ uint32_t kv_used;memcpy(&kv_buf_size, inp, sizeof(kv_buf_size)); inp += sizeof(kv_buf_size);memcpy(&kv_head, inp, sizeof(kv_head)); inp += sizeof(kv_head);memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size);+ memcpy(&kv_used, inp, sizeof(kv_used)); inp += sizeof(kv_used);if (kv_buf_size) {GGML_ASSERT(kv_self.buf.size == kv_buf_size);- const size_t elt_size = ggml_element_size(kv_self.k);+ const size_t elt_size = ggml_element_size(kv_self.k_l[0]);- ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true });- ggml_cgraph gf{};+ ggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true });+ ggml_cgraph * gf = ggml_new_graph(cpy_ctx);- ggml_tensor * kin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_head, n_layer);- kin3d->data = (void *) inp;- inp += ggml_nbytes(kin3d);+ for (int il = 0; il < n_layer; ++il) {+ ggml_tensor * kin2d = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd, kv_head);+ kin2d->data = (void *) inp;+ inp += ggml_nbytes(kin2d);- ggml_tensor * vin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_head, n_embd, n_layer);- vin3d->data = (void *) inp;- inp += ggml_nbytes(vin3d);+ ggml_tensor * vin2d = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd);+ vin2d->data = (void *) inp;+ inp += ggml_nbytes(vin2d);- ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k,- n_embd, kv_head, n_layer,- elt_size*n_embd, elt_size*n_embd*n_ctx, 0);+ ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il],+ n_embd, kv_head,+ elt_size*n_embd, 0);- ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v,- kv_head, n_embd, n_layer,- elt_size*n_ctx, elt_size*n_ctx*n_embd, 0);+ ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il],+ kv_head, n_embd,+ elt_size*n_ctx, 0);++ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, kin2d, k2d));+ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, vin2d, v2d));+ }- ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, kin3d, k3d));- ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, vin3d, v3d));- ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1);+ ggml_graph_compute_helper(ctx->work_buffer, gf, /*n_threads*/ 1);ggml_free(cpy_ctx);}ctx->kv_self.head = kv_head;ctx->kv_self.size = kv_size;+ ctx->kv_self.used = kv_used;ctx->kv_self.cells.resize(kv_size);@@ -9119,6 +9984,14 @@ llama_token llama_token_nl(const struct llama_model * model) {return model->vocab.linefeed_id;}+int llama_add_bos_token(const struct llama_model * model) {+ return model->vocab.special_add_bos;+}++int llama_add_eos_token(const struct llama_model * model) {+ return model->vocab.special_add_eos;+}+llama_token llama_token_prefix(const struct llama_model * model) {return model->vocab.special_prefix_id;}diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 4f05ff16..405f9dcb 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -1,13 +1,12 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#ifndef LLAMA_H#define LLAMA_H#include "ggml.h"-#ifdef GGML_USE_CUBLAS#include "ggml-cuda.h"#define LLAMA_MAX_DEVICES GGML_CUDA_MAX_DEVICES-#else-#define LLAMA_MAX_DEVICES 1-#endif // GGML_USE_CUBLAS#include <stddef.h>#include <stdint.h>#include <stdio.h>@@ -42,7 +41,10 @@#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN-#define LLAMA_SESSION_VERSION 2+#define LLAMA_SESSION_VERSION 3++// Defined when llama.cpp is compiled with support for offloading model layers to GPU.+#define LLAMA_SUPPORTS_GPU_OFFLOAD#ifdef __cplusplusextern "C" {@@ -153,6 +155,22 @@ extern "C" {llama_seq_id all_seq_id; // used if seq_id == NULL} llama_batch;+ enum llama_model_kv_override_type {+ LLAMA_KV_OVERRIDE_INT,+ LLAMA_KV_OVERRIDE_FLOAT,+ LLAMA_KV_OVERRIDE_BOOL,+ };++ struct llama_model_kv_override {+ char key[128];+ enum llama_model_kv_override_type tag;+ union {+ int64_t int_value;+ double float_value;+ bool bool_value;+ };+ };+struct llama_model_params {int32_t n_gpu_layers; // number of layers to store in VRAMint32_t main_gpu; // the GPU that is used for scratch and small tensors@@ -160,9 +178,13 @@ extern "C" {// called with a progress value between 0 and 1, pass NULL to disablellama_progress_callback progress_callback;+// context pointer passed to the progress callbackvoid * progress_callback_user_data;+ // override key-value pairs of the model meta data+ const struct llama_model_kv_override * kv_overrides;+// Keep the booleans together to avoid misalignment during copy-by-value.bool vocab_only; // only load the vocabulary, no weightsbool use_mmap; // use mmap if possible@@ -180,17 +202,20 @@ extern "C" {// ref: https://github.com/ggerganov/llama.cpp/pull/2054float rope_freq_base; // RoPE base frequency, 0 = from modelfloat rope_freq_scale; // RoPE frequency scaling factor, 0 = from model- float yarn_ext_factor; // YaRN extrapolation mix factor, NaN = from model+ float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from modelfloat yarn_attn_factor; // YaRN magnitude scaling factorfloat yarn_beta_fast; // YaRN low correction dimfloat yarn_beta_slow; // YaRN high correction dimuint32_t yarn_orig_ctx; // YaRN original context size+ enum ggml_type type_k; // data type for K cache+ enum ggml_type type_v; // data type for V cache+// Keep the booleans together to avoid misalignment during copy-by-value.- bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true)- bool f16_kv; // use fp16 for KV cache, fp32 otherwise- bool logits_all; // the llama_eval() call computes all logits, not just the last one- bool embedding; // embedding mode only+ bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true)+ bool logits_all; // the llama_eval() call computes all logits, not just the last one+ bool embedding; // embedding mode only+ bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU};// model quantization parameters@@ -296,6 +321,23 @@ extern "C" {// Get the model's RoPE frequency scaling factorLLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);+ // Functions to access the model's GGUF metadata scalar values+ // - The functions return the length of the string on success, or -1 on failure+ // - The output string is always null-terminated and cleared on failure+ // - GGUF array values are not supported by these functions++ // Get metadata value as a string by key name+ LLAMA_API int llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);++ // Get the number of metadata key/value pairs+ LLAMA_API int llama_model_meta_count(const struct llama_model * model);++ // Get metadata key name by index+ LLAMA_API int llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size);++ // Get metadata value as a string by index+ LLAMA_API int llama_model_meta_val_str_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size);+// Get a string describing the model typeLLAMA_API int llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);@@ -339,9 +381,60 @@ extern "C" {// KV cache//- // Returns the number of tokens in the KV cache- LLAMA_API DEPRECATED(int llama_get_kv_cache_token_count(const struct llama_context * ctx),- "avoid using this, it will be removed in the future, instead - count the tokens in user code");+ // Information associated with an individual cell in the KV cache view.+ struct llama_kv_cache_view_cell {+ // The position for this cell. Takes KV cache shifts into account.+ // May be negative if the cell is not populated.+ llama_pos pos;+ };++ // An updateable view of the KV cache.+ struct llama_kv_cache_view {+ // Number of KV cache cells. This will be the same as the context size.+ int32_t n_cells;++ // Maximum number of sequences that can exist in a cell. It's not an error+ // if there are more sequences in a cell than this value, however they will+ // not be visible in the view cells_sequences.+ int32_t n_max_seq;++ // Number of tokens in the cache. For example, if there are two populated+ // cells, the first with 1 sequence id in it and the second with 2 sequence+ // ids then you'll have 3 tokens.+ int32_t token_count;++ // Number of populated cache cells.+ int32_t used_cells;++ // Maximum contiguous empty slots in the cache.+ int32_t max_contiguous;++ // Index to the start of the max_contiguous slot range. Can be negative+ // when cache is full.+ int32_t max_contiguous_idx;++ // Information for an individual cell.+ struct llama_kv_cache_view_cell * cells;++ // The sequences for each cell. There will be n_max_seq items per cell.+ llama_seq_id * cells_sequences;+ };++ // Create an empty KV cache view. (use only for debugging purposes)+ LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq);++ // Free a KV cache view. (use only for debugging purposes)+ LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view * view);++ // Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes)+ LLAMA_API void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view);++ // Returns the number of tokens in the KV cache (slow, use only for debug)+ // If a KV cell has multiple sequences assigned to it, it will be counted multiple times+ LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context * ctx);++ // Returns the number of used KV cells (i.e. have at least one sequence assigned to them)+ LLAMA_API int llama_get_kv_cache_used_cells(const struct llama_context * ctx);// Clear the KV cacheLLAMA_API void llama_kv_cache_clear(@@ -512,6 +605,12 @@ extern "C" {LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentenceLLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line+ // Returns -1 if unknown, 1 for true or 0 for false.+ LLAMA_API int llama_add_bos_token(const struct llama_model * model);++ // Returns -1 if unknown, 1 for true or 0 for false.+ LLAMA_API int llama_add_eos_token(const struct llama_model * model);+// codellama infill tokensLLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefixLLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middlediff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex 0bf5fd32..7bc674df 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -673,7 +673,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {// measure mem requirement and allocate{static const size_t tensor_alignment = 32;- new_clip->buf_compute.resize(ggml_tensor_overhead()*GGML_MAX_NODES + ggml_graph_overhead());+ new_clip->buf_compute.resize(ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead());new_clip->alloc = ggml_allocr_new_measure(tensor_alignment);clip_image_f32_batch batch;batch.size = 1;diff --git a/llama.cpp/llava/convert-image-encoder-to-gguf.py b/llama.cpp/llava/convert-image-encoder-to-gguf.pyindex 2f5eef19..66784d19 100644--- a/llama.cpp/llava/convert-image-encoder-to-gguf.py+++ b/llama.cpp/llava/convert-image-encoder-to-gguf.py@@ -51,7 +51,7 @@ def bytes_to_unicode():The reversible bpe codes work on unicode strings.This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.- This is a signficant percentage of your normal, say, 32K bpe vocab.+ This is a significant percentage of your normal, say, 32K bpe vocab.To avoid that, we want lookup tables between utf-8 bytes and unicode strings.And avoids mapping to whitespace/control characters the bpe code barfs on."""diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index 02c5ebb2..a293fe88 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -149,8 +149,6 @@ Ignore end of stream token and continue generating (implies.Fl Fl logit-bias Ar 2-inf ).It Fl Fl no-penalize-nlDo not penalize newline token.-.It Fl Fl memory-f32-Use f32 instead of f16 for memory key+value (default: disabled) Not recommended: doubles context memory required and no measurable increase in quality..It Fl Fl temp Ar NTemperature (default: 0.8).It Fl Fl logits-all@@ -211,6 +209,22 @@ Model path in GGUF file format (default: models/7B/ggml-model-f16.gguf)Draft model for speculative decoding (default: models/7B/ggml-model-f16.gguf).It Fl Fl unsecureDisables pledge() sandboxing on Linux and OpenBSD.+.It Fl Fl samplers+Samplers that will be used for generation in the order, separated by+semicolon, for example: top_k;tfs;typical;top_p;min_p;temp+.It Fl Fl samplers-seq+Simplified sequence for samplers that will be used.+.It Fl cml , Fl Fl chatml+Run in chatml mode (use with ChatML-compatible models)+.It Fl dkvc , Fl Fl dump-kv-cache+Verbose print of the KV cache.+.It Fl nkvo , Fl Fl no-kv-offload+Disable KV offload.+.It Fl ctk Ar TYPE , Fl Fl cache-type-k Ar TYPE+KV cache data type for K.+.It Fl ctv Ar TYPE , Fl Fl cache-type-v Ar TYPE+KV cache data type for V.+Disable KV offload..El.Sh LOG OPTIONSThe following log options are available:diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex e66ee094..e2144f23 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -1,49 +1,38 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#include <cosmo.h>-#include "llama.cpp/ggml-metal.h"-#include "llama.cpp/ggml-cuda.h"-#include "llamafile/version.h"-#include "llama.cpp/llava/llava.h"-#include "tool/args/args.h"--#include "llama.cpp/common.h"-#include "llama.cpp/console.h"-#include "llama.cpp/llama.h"-+#include <cassert>#include <cinttypes>#include <cmath>+#include <cosmo.h>#include <cstdio>#include <cstring>#include <ctime>#include <fstream>#include <iostream>+#include <signal.h>#include <sstream>#include <string>-#include <vector>-#include <signal.h>#include <unistd.h>+#include <vector>++#include "tool/args/args.h"+#include "llamafile/version.h"+#include "llama.cpp/llama.h"+#include "llama.cpp/common.h"+#include "llama.cpp/console.h"+#include "llama.cpp/ggml-cuda.h"+#include "llama.cpp/ggml-metal.h"+#include "llama.cpp/llava/llava.h"-static bool is_terminal;-static gpt_params * g_params;-static volatile bool is_terminated;-static volatile bool is_interacting;+static llama_context ** g_ctx;+static llama_model ** g_model;+static gpt_params * g_params;+static std::vector<llama_token> * g_input_tokens;+static std::ostringstream * g_output_ss;+static std::vector<llama_token> * g_output_tokens;+static bool is_interacting = false;-static void sigint_handler(int signo) {- if (g_params->interactive && !is_interacting) {- // if we're in interactive mode, ctrl-c may be used to get the- // llm to stop generating text and start accepting input again- is_interacting = true;- } else {- // 1. if we're in interactive mode, then pressing ctrl-c when- // waiting for user input will end the program like an eof- // 2. if we're using a purely generative mode then ctrl-c can- // be used to gracefully halt the text generation process.- is_terminated = true;- ggml_interrupt(true);- }-}static void write_logfile(const llama_context * ctx, const gpt_params & params, const llama_model * model,@@ -89,6 +78,26 @@ static void write_logfile(fclose(logfile);}+static void sigint_handler(int signo) {+ if (signo == SIGINT) {+ if (g_params->interactive && !is_interacting) {+ is_interacting = true;+ } else {+ console::cleanup();+ printf("\n");+ llama_print_timings(*g_ctx);+ write_logfile(*g_ctx, *g_params, *g_model, *g_input_tokens, g_output_ss->str(), *g_output_tokens);+ _exit(130);+ }+ }+}++static void llama_log_callback_logTee(ggml_log_level level, const char * text, void * user_data) {+ (void) level;+ (void) user_data;+ LOG_TEE("%s", text);+}+static bool has_argument(int argc, char ** argv, const char * arg) {for (int i = 1; i < argc; ++i) {if (!strcmp(argv[i], arg)) {@@ -128,6 +137,7 @@ int main(int argc, char ** argv) {log_set_target(log_filename_generator("main", "log"));LOG_TEE("Log start\n");log_dump_cmdline(argc, argv);+ llama_log_set(llama_log_callback_logTee, nullptr);#endif // LOG_DISABLE_LOGS// TODO: Dump params ?@@ -137,7 +147,6 @@ int main(int argc, char ** argv) {// (note for later: this is a slightly awkward choice)console::init(params.simple_io, params.use_color);atexit([]() { console::cleanup(); });- is_terminal = isatty(2);if (!params.unsecure && !ggml_metal_supported() && !ggml_cuda_supported()) {// Enable pledge() security on Linux and OpenBSD.@@ -210,6 +219,8 @@ int main(int argc, char ** argv) {llama_model * model;llama_context * ctx;llama_context * ctx_guidance = NULL;+ g_model = &model;+ g_ctx = &ctx;// load the model and apply lora adapter, if anyLOG("%s: load the model and apply lora adapter, if any\n", __func__);@@ -265,13 +276,16 @@ int main(int argc, char ** argv) {}}- const bool add_bos = llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM;+ const bool add_bos = llama_should_add_bos_token(model);LOG("add_bos: %d\n", add_bos);std::vector<llama_token> embd_inp;- if (params.interactive_first || params.instruct || !params.prompt.empty() || session_tokens.empty()) {+ if (params.interactive_first || params.instruct || params.chatml || !params.prompt.empty() || session_tokens.empty()) {LOG("tokenize the prompt\n");+ if (params.chatml) {+ params.prompt = "<|im_start|>system\n" + params.prompt + "<|im_end|>";+ }embd_inp = ::llama_tokenize(ctx, params.prompt, add_bos, true);} else {LOG("use session tokens\n");@@ -349,7 +363,7 @@ int main(int argc, char ** argv) {}// number of tokens to keep when resetting context- if (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct) {+ if (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct || params.chatml) {params.n_keep = (int)embd_inp.size();}@@ -360,11 +374,23 @@ int main(int argc, char ** argv) {LOG("inp_pfx: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, inp_pfx).c_str());LOG("inp_sfx: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, inp_sfx).c_str());+ // chatml prefix & suffix+ const auto cml_pfx = ::llama_tokenize(ctx, "\n<|im_start|>user\n", add_bos, true);+ const auto cml_sfx = ::llama_tokenize(ctx, "<|im_end|>\n<|im_start|>assistant\n", false, true);++ LOG("cml_pfx: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, cml_pfx).c_str());+ LOG("cml_sfx: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, cml_sfx).c_str());+// in instruct mode, we inject a prefix and a suffix to each input by the userif (params.instruct) {params.interactive_first = true;params.antiprompt.push_back("### Instruction:\n\n");}+ // similar for chatml mode+ else if (params.chatml) {+ params.interactive_first = true;+ params.antiprompt.push_back("<|im_start|>user\n");+ }// enable interactive mode if interactive start is specifiedif (params.interactive_first) {@@ -444,6 +470,7 @@ int main(int argc, char ** argv) {}}LOG_TEE("sampling: \n%s\n", llama_sampling_print(sparams).c_str());+ LOG_TEE("sampling order: \n%s\n", llama_sampling_order_print(sparams).c_str());LOG_TEE("generate: n_ctx = %d, n_batch = %d, n_predict = %d, n_keep = %d\n", n_ctx, params.n_batch, params.n_predict, params.n_keep);LOG_TEE("\n\n");@@ -476,9 +503,9 @@ int main(int argc, char ** argv) {int n_session_consumed = 0;int n_past_guidance = 0;- std::vector<int> input_tokens;- std::vector<int> output_tokens;- std::ostringstream output_ss;+ std::vector<int> input_tokens; g_input_tokens = &input_tokens;+ std::vector<int> output_tokens; g_output_tokens = &output_tokens;+ std::ostringstream output_ss; g_output_ss = &output_ss;// the first thing we will do is to output the prompt, so set color accordinglyconsole::set_display(console::prompt);@@ -488,7 +515,7 @@ int main(int argc, char ** argv) {struct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams);- while (!is_terminated && ((n_remain != 0 && !is_antiprompt) || params.interactive)) {+ while ((n_remain != 0 && !is_antiprompt) || params.interactive) {// predictif (!embd.empty()) {// Note: n_ctx - 4 here is to match the logic for commandline prompt handling via@@ -734,7 +761,7 @@ int main(int argc, char ** argv) {is_interacting = true;printf("\n");- } else if (params.instruct) {+ } else if (params.instruct || params.chatml) {is_interacting = true;} else if (params.silent_prompt) {// --silent-prompt usually goes with `./main 2>/dev/null` and@@ -747,7 +774,7 @@ int main(int argc, char ** argv) {if (n_past > 0 && is_interacting) {LOG("waiting for user input\n");- if (params.instruct) {+ if (params.instruct || params.chatml) {printf("\n> ");}@@ -794,6 +821,12 @@ int main(int argc, char ** argv) {n_consumed = embd_inp.size();embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end());}+ // chatml mode: insert user chat prefix+ if (params.chatml && !is_antiprompt) {+ LOG("inserting chatml prefix\n");+ n_consumed = embd_inp.size();+ embd_inp.insert(embd_inp.end(), cml_pfx.begin(), cml_pfx.end());+ }if (params.escape) {process_escapes(buffer);}@@ -812,6 +845,11 @@ int main(int argc, char ** argv) {LOG("inserting instruction suffix\n");embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end());}+ // chatml mode: insert assistant chat suffix+ if (params.chatml) {+ LOG("inserting chatml suffix\n");+ embd_inp.insert(embd_inp.end(), cml_sfx.begin(), cml_sfx.end());+ }for (size_t i = original_size; i < embd_inp.size(); ++i) {const llama_token token = embd_inp[i];@@ -837,7 +875,7 @@ int main(int argc, char ** argv) {}// end of text token- if (!embd.empty() && embd.back() == llama_token_eos(model) && !(params.instruct || params.interactive)) {+ if (!embd.empty() && embd.back() == llama_token_eos(model) && !(params.instruct || params.interactive || params.chatml)) {LOG_TEE(" [end of text]\n");break;}diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex 901141c5..30f2c608 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -1,5 +1,6 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "llama.cpp/common.h"#include "llama.cpp/llama.h"#include "llamafile/version.h"diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cppindex 1317024c..5a6cef06 100644--- a/llama.cpp/sampling.cpp+++ b/llama.cpp/sampling.cpp@@ -1,3 +1,6 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "sampling.h"struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_params & params) {@@ -99,6 +102,56 @@ std::string llama_sampling_print(const llama_sampling_params & params) {return std::string(result);}+std::string llama_sampling_order_print(const llama_sampling_params & params) {+ std::string result = "CFG -> Penalties ";+ if (params.mirostat == 0) {+ for (auto s : params.samplers_sequence) {+ switch (s) {+ case 'k': result += "-> top_k "; break;+ case 'f': result += "-> tfs_z "; break;+ case 'y': result += "-> typical_p "; break;+ case 'p': result += "-> top_p "; break;+ case 'm': result += "-> min_p "; break;+ case 't': result += "-> temp "; break;+ default : break;+ }+ }+ } else {+ result += "-> mirostat ";+ }++ return result;+}++// no reasons to expose this function in header+static void sampler_queue(+ struct llama_context * ctx_main,+ const llama_sampling_params & params,+ llama_token_data_array & cur_p,+ size_t & min_keep) {+ const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));++ const float temp = params.temp;+ const int32_t top_k = params.top_k <= 0 ? n_vocab : params.top_k;+ const float top_p = params.top_p;+ const float min_p = params.min_p;+ const float tfs_z = params.tfs_z;+ const float typical_p = params.typical_p;+ const std::string & samplers_sequence = params.samplers_sequence;++ for (auto s : samplers_sequence) {+ switch (s){+ case 'k': llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep); break;+ case 'f': llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep); break;+ case 'y': llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break;+ case 'p': llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break;+ case 'm': llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break;+ case 't': llama_sample_temp (ctx_main, &cur_p, temp); break;+ default : break;+ }+ }+}+llama_token llama_sampling_sample(struct llama_sampling_context * ctx_sampling,struct llama_context * ctx_main,@@ -109,11 +162,6 @@ llama_token llama_sampling_sample(const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));const float temp = params.temp;- const int32_t top_k = params.top_k <= 0 ? n_vocab : params.top_k;- const float top_p = params.top_p;- const float min_p = params.min_p;- const float tfs_z = params.tfs_z;- const float typical_p = params.typical_p;const int32_t penalty_last_n = params.penalty_last_n < 0 ? params.n_prev : params.penalty_last_n;const float penalty_repeat = params.penalty_repeat;const float penalty_freq = params.penalty_freq;@@ -188,12 +236,7 @@ llama_token llama_sampling_sample(// temperature samplingsize_t min_keep = std::max(1, params.n_probs);- llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep);- llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep);- llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep);- llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep);- llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep);- llama_sample_temp (ctx_main, &cur_p, temp);+ sampler_queue(ctx_main, params, cur_p, min_keep);id = llama_sample_token(ctx_main, &cur_p);diff --git a/llama.cpp/sampling.h b/llama.cpp/sampling.hindex 7c9b8dcf..fdfa9eed 100644--- a/llama.cpp/sampling.h+++ b/llama.cpp/sampling.h@@ -10,22 +10,23 @@// sampling parameterstypedef struct llama_sampling_params {- int32_t n_prev = 64; // number of previous tokens to remember- int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.- int32_t top_k = 40; // <= 0 to use vocab size- float top_p = 0.95f; // 1.0 = disabled- float min_p = 0.05f; // 0.0 = disabled- float tfs_z = 1.00f; // 1.0 = disabled- float typical_p = 1.00f; // 1.0 = disabled- float temp = 0.80f; // 1.0 = disabled- int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)- float penalty_repeat = 1.10f; // 1.0 = disabled- float penalty_freq = 0.00f; // 0.0 = disabled- float penalty_present = 0.00f; // 0.0 = disabled- int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0- float mirostat_tau = 5.00f; // target entropy- float mirostat_eta = 0.10f; // learning rate- bool penalize_nl = true; // consider newlines as a repeatable token+ int32_t n_prev = 64; // number of previous tokens to remember+ int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.+ int32_t top_k = 40; // <= 0 to use vocab size+ float top_p = 0.95f; // 1.0 = disabled+ float min_p = 0.05f; // 0.0 = disabled+ float tfs_z = 1.00f; // 1.0 = disabled+ float typical_p = 1.00f; // 1.0 = disabled+ float temp = 0.80f; // 1.0 = disabled+ int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)+ float penalty_repeat = 1.10f; // 1.0 = disabled+ float penalty_freq = 0.00f; // 0.0 = disabled+ float penalty_present = 0.00f; // 0.0 = disabled+ int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0+ float mirostat_tau = 5.00f; // target entropy+ float mirostat_eta = 0.10f; // learning rate+ bool penalize_nl = true; // consider newlines as a repeatable token+ std::string samplers_sequence = "kfypmt"; // top_k, tail_free, typical_p, top_p, min_p, tempstd::string grammar; // optional BNF-like grammar to constrain sampling@@ -80,6 +81,9 @@ std::string llama_sampling_prev_str(llama_sampling_context * ctx_sampling, llama// Print sampling parameters into a stringstd::string llama_sampling_print(const llama_sampling_params & params);+// Print sampling order into a string+std::string llama_sampling_order_print(const llama_sampling_params & params);+// this is a common sampling function used across the examples for convenience// it can serve as a starting point for implementing your own sampling function// Note: When using multiple sequences, it is the caller's responsibility to calldiff --git a/llama.cpp/server/README.md b/llama.cpp/server/README.mdindex cfc220f5..0751b961 100644--- a/llama.cpp/server/README.md+++ b/llama.cpp/server/README.md@@ -222,7 +222,7 @@ node index.js`content`: Set the text to process.- **POST** `/infill`: For code infilling. Takes a prefix and a suffix and returns the predicted completion as stream.+- **POST** `/infill`: For code infilling. Takes a prefix and a suffix and returns the predicted completion as stream.*Options:*diff --git a/llama.cpp/server/json.h b/llama.cpp/server/json.hindex 4d1a37ad..ea945f34 100644--- a/llama.cpp/server/json.h+++ b/llama.cpp/server/json.h@@ -11227,7 +11227,7 @@ class binary_reader}if (is_ndarray) // ndarray dimensional vector can only contain integers, and can not embed another array{- return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimentional vector is not allowed", "size"), nullptr));+ return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr));}std::vector<size_t> dim;if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim)))diff --git a/llama.cpp/server/public/completion.js b/llama.cpp/server/public/completion.jsindex 0c9bd5f1..6d6ffaed 100644--- a/llama.cpp/server/public/completion.js+++ b/llama.cpp/server/public/completion.js@@ -110,7 +110,7 @@ export async function* llama(prompt, params = {}, config = {}) {return content;}-// Call llama, return an event target that you can subcribe to+// Call llama, return an event target that you can subscribe to//// Example://diff --git a/llama.cpp/server/public/index.html b/llama.cpp/server/public/index.htmlindex c916d012..24779868 100644--- a/llama.cpp/server/public/index.html+++ b/llama.cpp/server/public/index.html@@ -243,7 +243,7 @@cache_prompt: true})- /* START: Support for storing prompt templates and parameters in borwser LocalStorage */+ /* START: Support for storing prompt templates and parameters in browsers LocalStorage */const local_storage_storageKey = "llamacpp_server_local_storage";@@ -287,7 +287,7 @@let importedTemplates = local_storage_getDataAsObject('user_templates')if (importedTemplates) {- // saved templates were successfuly imported.+ // saved templates were successfully imported.console.log('Processing saved templates and updating default template')params.value = { ...params.value, image_data: [] };@@ -308,7 +308,7 @@}function userTemplateResetToDefault() {- console.log('Reseting themplate to default')+ console.log('Resetting template to default')selectedUserTemplate.value.name = 'default';selectedUserTemplate.value.data = savedUserTemplates.value['default'];}@@ -750,7 +750,7 @@<fieldset class="two">${IntField({ label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict })}- ${FloatField({ label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}+ ${FloatField({ label: "Temperature", max: 2.0, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}${FloatField({ label: "Penalize repeat sequence", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}${IntField({ label: "Consider N tokens for penalize", max: 2048, min: 0, name: "repeat_last_n", value: params.value.repeat_last_n })}${IntField({ label: "Top-K sampling", max: 100, min: -1, name: "top_k", value: params.value.top_k })}diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 1a3d66ec..1a3fc2ab 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -1,6 +1,5 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi-#include <cosmo.h>+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#include "tool/args/args.h"#include "llama.cpp/common.h"#include "llama.cpp/llama.h"@@ -17,6 +16,7 @@#include "httplib.h"#include "json.h"+#include <cosmo.h>#include <unistd.h>#include <cstdio>#include <cstddef>@@ -523,6 +523,7 @@ struct llama_server_contextbool multimodal = false;bool clean_kv_cache = true;bool all_slots_are_idle = false;+ bool add_bos_token = true;int32_t id_gen;int32_t n_ctx; // total context for all clients / slots@@ -598,6 +599,8 @@ struct llama_server_contextn_ctx = llama_n_ctx(ctx);+ add_bos_token = llama_should_add_bos_token(model);+return true;}@@ -631,6 +634,11 @@ struct llama_server_contextstd::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const{+ // TODO: currently, we tokenize using special tokens by default+ // this is not always correct (see https://github.com/ggerganov/llama.cpp/pull/4160#issuecomment-1824826216)+ // but it's better compared to completely ignoring ChatML and other chat templates+ const bool TMP_FORCE_SPECIAL = true;+// If `add_bos` is true, we only add BOS, when json_prompt is a string,// or the first element of the json_prompt array is a string.std::vector<llama_token> prompt_tokens;@@ -646,12 +654,12 @@ struct llama_server_contextstd::vector<llama_token> p;if (first){- p = ::llama_tokenize(ctx, s, add_bos);+ p = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);first = false;}else{- p = ::llama_tokenize(ctx, s, false);+ p = ::llama_tokenize(ctx, s, false, TMP_FORCE_SPECIAL);}prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());}@@ -668,7 +676,7 @@ struct llama_server_contextelse{auto s = json_prompt.template get<std::string>();- prompt_tokens = ::llama_tokenize(ctx, s, add_bos);+ prompt_tokens = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);}return prompt_tokens;@@ -712,6 +720,7 @@ struct llama_server_contextslot->params.n_predict = json_value(data, "n_predict", default_params.n_predict);slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);+ slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);@@ -900,7 +909,7 @@ struct llama_server_context}void update_system_prompt() {- system_tokens = ::llama_tokenize(ctx, system_prompt, true);+ system_tokens = ::llama_tokenize(ctx, system_prompt, add_bos_token);llama_batch_clear(batch);@@ -1143,6 +1152,7 @@ struct llama_server_contextmulti.id = id;std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));queue_multitasks.push_back(multi);+ condition_tasks.notify_one();}void update_multi_task(int multitask_id, int subtask_id, task_result& result)@@ -1154,6 +1164,7 @@ struct llama_server_context{multitask.subtasks_remaining.erase(subtask_id);multitask.results.push_back(result);+ condition_tasks.notify_one();}}}@@ -1175,6 +1186,7 @@ struct llama_server_context{"temp", slot.sparams.temp},{"top_k", slot.sparams.top_k},{"top_p", slot.sparams.top_p},+ {"min_p", slot.sparams.min_p},{"tfs_z", slot.sparams.tfs_z},{"typical_p", slot.sparams.typical_p},{"repeat_last_n", slot.sparams.penalty_last_n},@@ -1563,6 +1575,7 @@ struct llama_server_contextstd::lock_guard<std::mutex> lock(mutex_results);queue_results.push_back(aggregate_result);+ condition_results.notify_all();queue_iterator = queue_multitasks.erase(queue_iterator);}@@ -1711,7 +1724,7 @@ struct llama_server_context}else{- prompt_tokens = tokenize(slot.prompt, system_prompt.empty()); // add BOS if there isn't system prompt+ prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt}slot.num_prompt_tokens = prompt_tokens.size();@@ -1788,7 +1801,7 @@ struct llama_server_contextconst bool has_images = process_images(slot);// process the prefix of first image- std::vector<llama_token> prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, true) : prompt_tokens;+ std::vector<llama_token> prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens;for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past){llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot.n_past, { slot.id }, false);@@ -2111,10 +2124,6 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}params.yarn_beta_slow = std::stof(argv[i]);}- else if (arg == "--memory-f32" || arg == "--memory_f32")- {- params.memory_f16 = false;- }else if (arg == "--threads" || arg == "-t"){if (++i >= argc)@@ -2637,6 +2646,7 @@ int main(int argc, char **argv)llamafile_check_cpu();LoadZipArgs(&argc, &argv);+ ShowCrashReports();// own arguments required by this examplegpt_params params;@@ -2678,12 +2688,6 @@ int main(int argc, char **argv){"Access-Control-Allow-Origin", "*"},{"Access-Control-Allow-Headers", "content-type"}});- svr.Get("/statusz", [](const httplib::Request &, httplib::Response &res)- {- res.set_content("todo\r\n", strlen("todo\r\n"), "text/html");- return false;- });-svr.Get("/props", [&llama](const httplib::Request & /*req*/, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", "*");@@ -2732,6 +2736,17 @@ int main(int argc, char **argv)break;}} else {+ const std::string str =+ "error: " ++ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";+ LOG_VERBOSE("data stream", {+ { "to_send", str }+ });+ if (!sink.write(str.c_str(), str.size()))+ {+ return false;+ }break;}}@@ -2865,9 +2880,9 @@ int main(int argc, char **argv)task_result result = llama.next_result(task_id);if (!result.error) {const std::string str =- "data: " +- result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";+ "data: " ++ result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";LOG_VERBOSE("data stream", {{ "to_send", str }});diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 9d50962b..5082f51c 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -33,10 +33,13 @@__static_yoink("llama.cpp/ggml.h");__static_yoink("llamafile/compcap.cu");+__static_yoink("llama.cpp/ggml-impl.h");__static_yoink("llamafile/llamafile.h");__static_yoink("llama.cpp/ggml-cuda.h");+__static_yoink("llama.cpp/ggml-alloc.h");__static_yoink("llama.cpp/ggml-cuda.cu");-__static_yoink("llama.cpp/ggml-cuda.dll");+__static_yoink("llama.cpp/ggml-backend.h");+__static_yoink("llama.cpp/ggml-backend-impl.h");#define NVCC_LIBS "-lcublas"@@ -60,7 +63,11 @@ static const struct Source {{"/zip/llama.cpp/ggml.h", "ggml.h"},{"/zip/llamafile/compcap.cu", "compcap.cu"},{"/zip/llamafile/llamafile.h", "llamafile.h"},+ {"/zip/llama.cpp/ggml-impl.h", "ggml-impl.h"},{"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},+ {"/zip/llama.cpp/ggml-alloc.h", "ggml-alloc.h"},+ {"/zip/llama.cpp/ggml-backend.h", "ggml-backend.h"},+ {"/zip/llama.cpp/ggml-backend-impl.h", "ggml-backend-impl.h"},{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last};@@ -88,6 +95,8 @@ static struct Cuda {typeof(ggml_cuda_compute_forward) *compute_forward;typeof(ggml_cuda_get_device_count) *get_device_count;typeof(ggml_cuda_get_device_description) *get_device_description;+ typeof(ggml_backend_reg_cuda_init) *reg_cuda_init;+ typeof(ggml_backend_cuda_buffer_type) *buffer_type;} ggml_cuda;static const char *Dlerror(void) {@@ -440,6 +449,8 @@ static bool LinkCudaDso(const char *dso) {ok &= !!(ggml_cuda.compute_forward = cosmo_dlsym(lib, "ggml_cuda_compute_forward"));ok &= !!(ggml_cuda.get_device_count = cosmo_dlsym(lib, "ggml_cuda_get_device_count"));ok &= !!(ggml_cuda.get_device_description = cosmo_dlsym(lib, "ggml_cuda_get_device_description"));+ ok &= !!(ggml_cuda.reg_cuda_init = cosmo_dlsym(lib, "ggml_backend_reg_cuda_init"));+ ok &= !!(ggml_cuda.buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_buffer_type"));if (!ok) {tinyprint(2, Dlerror(), ": not all symbols could be imported\n", NULL);return false;@@ -592,3 +603,14 @@ void ggml_cuda_get_device_description(int device,return ggml_cuda.get_device_description(device, description,description_size);}++ggml_backend_t ggml_backend_reg_cuda_init(const char * params,+ void * user_data) {+ if (!ggml_cuda_supported()) return 0;+ return ggml_cuda.reg_cuda_init(params, user_data);+}++ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {+ if (!ggml_cuda_supported()) return 0;+ return ggml_cuda.buffer_type(device);+}diff --git a/llamafile/metal.c b/llamafile/metal.cindex a275e36a..6a0d9e93 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -33,10 +33,13 @@__static_yoink("llama.cpp/ggml.h");__static_yoink("llamafile/llamafile.h");__static_yoink("llama.cpp/ggml-impl.h");+__static_yoink("llama.cpp/ggml-alloc.h");__static_yoink("llama.cpp/ggml-metal.m");__static_yoink("llama.cpp/ggml-metal.h");__static_yoink("llama.cpp/ggml-quants.h");+__static_yoink("llama.cpp/ggml-backend.h");__static_yoink("llama.cpp/ggml-metal.metal");+__static_yoink("llama.cpp/ggml-backend-impl.h");static const struct Source {const char *zip;@@ -46,8 +49,11 @@ static const struct Source {{"/zip/llamafile/llamafile.h", "llamafile.h"},{"/zip/llama.cpp/ggml-impl.h", "ggml-impl.h"},{"/zip/llama.cpp/ggml-metal.h", "ggml-metal.h"},+ {"/zip/llama.cpp/ggml-alloc.h", "ggml-alloc.h"},{"/zip/llama.cpp/ggml-quants.h", "ggml-quants.h"},+ {"/zip/llama.cpp/ggml-backend.h", "ggml-backend.h"},{"/zip/llama.cpp/ggml-metal.metal", "ggml-metal.metal"},+ {"/zip/llama.cpp/ggml-backend-impl.h", "ggml-backend-impl.h"},{"/zip/llama.cpp/ggml-metal.m", "ggml-metal.m"}, // must come last};@@ -224,15 +230,9 @@ void ggml_metal_host_free(void *data) {return ggml_metal.host_free(data);}-struct ggml_metal_context *ggml_metal_init(int n_cb, const char *metalPath) {- char path[PATH_MAX];+struct ggml_metal_context *ggml_metal_init(int n_cb) {if (!ggml_metal_supported()) return NULL;- if (!metalPath) {- llamafile_get_app_dir(path, PATH_MAX);- strlcat(path, "ggml-metal.metal", sizeof(path));- metalPath = path;- }- return ggml_metal.init(n_cb, metalPath);+ return ggml_metal.init(n_cb);}bool ggml_metal_add_buffer(struct ggml_metal_context *ctx,