Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Synchronize with llama.cpp upstream
Justine Tunney5447ca811366
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex d164f0a1..2c0a8cb7 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -9,8 +9,8 @@ LICENSEORIGINhttps://github.com/ggerganov/llama.cpp/pull/4406/- 6db2b41a76ee78d5efdd5c3cddd5d7ad3f646855- 2024-01-27+ 973053d8b0d04809836b3339a50f68d9c842de90+ 2024-02-22LOCAL MODIFICATIONSdiff --git a/llama.cpp/base64.h b/llama.cpp/base64.hindex ab4775d3..563247a6 100644--- a/llama.cpp/base64.h+++ b/llama.cpp/base64.h@@ -235,7 +235,7 @@ public:++in_begin;if (c != '=') {- throw std::runtime_error("invalid base64 character.");+ throw base64_error("invalid base64 character.");}}}@@ -385,7 +385,7 @@ private:}}- throw std::runtime_error("invalid base64 character.");+ throw base64_error("invalid base64 character.");}};diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 12607944..fc600f15 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -344,13 +344,14 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {invalid_param = true;break;}- sparams.samplers_sequence = parse_samplers_input(argv[i]);+ const auto sampler_names = string_split(argv[i], ';');+ sparams.samplers_sequence = sampler_types_from_names(sampler_names, true);} else if (arg == "--sampling-seq") {if (++i >= argc) {invalid_param = true;break;}- sparams.samplers_sequence = argv[i];+ sparams.samplers_sequence = sampler_types_from_chars(argv[i]);} else if (arg == "--top-p") {if (++i >= argc) {invalid_param = true;@@ -407,6 +408,18 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {break;}sparams.penalty_present = std::stof(argv[i]);+ } else if (arg == "--dynatemp-range") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ sparams.dynatemp_range = std::stof(argv[i]);+ } else if (arg == "--dynatemp-exp") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ sparams.dynatemp_exponent = std::stof(argv[i]);} else if (arg == "--mirostat") {if (++i >= argc) {invalid_param = true;@@ -523,7 +536,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {invalid_param = true;break;}- params.lora_adapter.push_back(std::make_tuple(argv[i], 1.0f));+ params.lora_adapter.emplace_back(argv[i], 1.0f);params.use_mmap = false;} else if (arg == "--lora-scaled") {if (++i >= argc) {@@ -535,7 +548,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {invalid_param = true;break;}- params.lora_adapter.push_back(std::make_tuple(lora_adapter, std::stof(argv[i])));+ params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));params.use_mmap = false;} else if (arg == "--lora-base") {if (++i >= argc) {@@ -568,7 +581,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {} else if (arg == "--infill") {params.infill = true;} else if (arg == "--unsecure") {- params.unsecure = true;+ FLAG_unsecure = true;} else if (arg == "--nocompile") {FLAG_nocompile = true;} else if (arg == "--recompile") {@@ -627,6 +640,22 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {break;}params.main_gpu = std::stoi(argv[i]);+ } else if (arg == "--split-mode" || arg == "-sm") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ std::string arg_next = argv[i];+ if (arg_next == "none") {+ params.split_mode = LLAMA_SPLIT_NONE;+ } else if (arg_next == "layer") {+ params.split_mode = LLAMA_SPLIT_LAYER;+ } else if (arg_next == "row") {+ params.split_mode = LLAMA_SPLIT_ROW;+ } else {+ invalid_param = true;+ break;+ }} else if (arg == "--tensor-split" || arg == "-ts") {passed_gpu_flags = true;if (++i >= argc) {@@ -639,11 +668,11 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {const std::regex regex{R"([,/]+)"};std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1};std::vector<std::string> split_arg{it, {}};- if (split_arg.size() >= LLAMA_MAX_DEVICES) {+ if (split_arg.size() >= llama_max_devices()) {invalid_param = true;break;}- for (size_t i = 0; i < LLAMA_MAX_DEVICES; ++i) {+ for (size_t i = 0; i < llama_max_devices(); ++i) {if (i < split_arg.size()) {params.tensor_split[i] = std::stof(split_arg[i]);} else {@@ -653,7 +682,15 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {} else if (arg == "--no-mmap") {params.use_mmap = false;} else if (arg == "--numa") {- params.numa = true;+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ std::string value(argv[i]);+ /**/ if (value == "distribute" || value == "") { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }+ else { invalid_param = true; break; }} else if (arg == "--verbose-prompt") {params.verbose_prompt = true;} else if (arg == "--no-display-prompt" || arg == "--silent-prompt") {@@ -663,7 +700,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {invalid_param = true;break;}- params.antiprompt.push_back(argv[i]);+ params.antiprompt.emplace_back(argv[i]);} else if (arg == "-ld" || arg == "--logdir") {if (++i >= argc) {invalid_param = true;@@ -879,7 +916,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {}if (!params.kv_overrides.empty()) {- params.kv_overrides.emplace_back(llama_model_kv_override());+ params.kv_overrides.emplace_back();params.kv_overrides.back().key[0] = 0;}@@ -891,6 +928,14 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {const llama_sampling_params & sparams = params.sparams;+ std::string sampler_type_chars;+ std::string sampler_type_names;+ for (const auto sampler_type : sparams.samplers_sequence) {+ sampler_type_chars += static_cast<char>(sampler_type);+ sampler_type_names += sampler_type_to_name_string(sampler_type) + ";";+ }+ sampler_type_names.pop_back();+printf("\n");printf("usage: %s [options]\n", argv[0]);printf("\n");@@ -911,7 +956,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -tb N, --threads-batch N\n");printf(" number of threads to use during batch and prompt processing (default: same as --threads)\n");printf(" -td N, --threads-draft N");- printf(" number of threads to use during generation (default: same as --threads)");+ printf(" number of threads to use during generation (default: same as --threads)\n");printf(" -tbd N, --threads-batch-draft N\n");printf(" number of threads to use during batch and prompt processing (default: same as --threads-draft)\n");printf(" -p PROMPT, --prompt PROMPT\n");@@ -932,8 +977,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -n N, --n-predict N number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)\n", params.n_predict);printf(" -c N, --ctx-size N size of the prompt context (default: %d, 0 = loaded from model)\n", params.n_ctx);printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);- printf(" --samplers samplers that will be used for generation in the order, separated by \';\', for example: \"top_k;tfs;typical;top_p;min_p;temp\"\n");- printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sparams.samplers_sequence.c_str());+ printf(" --samplers samplers that will be used for generation in the order, separated by \';\'\n");+ printf(" (default: %s)\n", sampler_type_names.c_str());+ printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sampler_type_chars.c_str());printf(" --top-k N top-k sampling (default: %d, 0 = disabled)\n", sparams.top_k);printf(" --top-p N top-p sampling (default: %.1f, 1.0 = disabled)\n", (double)sparams.top_p);printf(" --min-p N min-p sampling (default: %.1f, 0.0 = disabled)\n", (double)sparams.min_p);@@ -943,6 +989,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --repeat-penalty N penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)\n", (double)sparams.penalty_repeat);printf(" --presence-penalty N repeat alpha presence penalty (default: %.1f, 0.0 = disabled)\n", (double)sparams.penalty_present);printf(" --frequency-penalty N repeat alpha frequency penalty (default: %.1f, 0.0 = disabled)\n", (double)sparams.penalty_freq);+ printf(" --dynatemp-range N dynamic temperature range (default: %.1f, 0.0 = disabled)\n", (double)sparams.dynatemp_range);+ printf(" --dynatemp-exp N dynamic temperature exponent (default: %.1f)\n", (double)sparams.dynatemp_exponent);printf(" --mirostat N use Mirostat sampling.\n");printf(" Top K, Nucleus, Tail Free and Locally Typical samplers are ignored if used.\n");printf(" (default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)\n", sparams.mirostat);@@ -979,7 +1027,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --winogrande-tasks N number of tasks to use when computing the Winogrande score (default: %zu)\n", params.winogrande_tasks);printf(" --multiple-choice compute multiple choice score over random tasks from datafile supplied with -f\n");printf(" --multiple-choice-tasks N number of tasks to use when computing the multiple choice score (default: %zu)\n", params.winogrande_tasks);- printf(" --kl-divergence computes KL-divergence to logits provided via --kl-divergence-base");+ printf(" --kl-divergence computes KL-divergence to logits provided via --kl-divergence-base\n");printf(" --keep N number of tokens to keep from the initial prompt (default: %d, -1 = all)\n", params.n_keep);printf(" --draft N number of tokens to draft for speculative decoding (default: %d)\n", params.n_draft);printf(" --chunks N max number of chunks to process (default: %d, -1 = all)\n", params.n_chunks);@@ -990,30 +1038,33 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA. see examples/llava/README.md\n");printf(" --image IMAGE_FILE path to an image file. use with multimodal models\n");- if (llama_mlock_supported()) {+ if (llama_supports_mlock()) {printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");}- if (llama_mmap_supported()) {+ if (llama_supports_mmap()) {printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");}- printf(" --numa attempt optimizations that help on some NUMA systems\n");+ printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n");+ printf(" - distribute: spread execution evenly over all nodes\n");+ printf(" - isolate: only spawn threads on CPUs on the node that execution started on\n");+ printf(" - numactl: use the CPU map provided by numactl\n");printf(" if run without this previously, it is recommended to drop the system page cache before using this\n");printf(" see https://github.com/ggerganov/llama.cpp/issues/1437\n");-#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD- printf(" -ngl N, --n-gpu-layers N\n");- printf(" number of layers to store in VRAM\n");- printf(" -ngld N, --n-gpu-layers-draft N\n");- printf(" number of layers to store in VRAM for the draft model\n");- printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");- printf(" how to split the model across multiple GPUs, one of:\n");- printf(" - none: use one GPU only\n");- printf(" - layer (default): split layers and KV across GPUs\n");- printf(" - row: split rows across GPUs\n");- printf(" -ts SPLIT, --tensor-split SPLIT\n");- printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");- printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");- printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu);-#endif+ if (llama_supports_gpu_offload()) {+ printf(" -ngl N, --n-gpu-layers N\n");+ printf(" number of layers to store in VRAM\n");+ printf(" -ngld N, --n-gpu-layers-draft N\n");+ printf(" number of layers to store in VRAM for the draft model\n");+ printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");+ printf(" how to split the model across multiple GPUs, one of:\n");+ printf(" - none: use one GPU only\n");+ printf(" - layer (default): split layers and KV across GPUs\n");+ printf(" - row: split rows across GPUs\n");+ printf(" -ts SPLIT, --tensor-split SPLIT\n");+ printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");+ printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");+ printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu);+ }printf(" --verbose-prompt print a verbose prompt before generation (default: %s)\n", params.verbose_prompt ? "true" : "false");printf(" --no-display-prompt don't print prompt at generation (default: %s)\n", !params.display_prompt ? "true" : "false");printf(" -gan N, --grp-attn-n N\n");@@ -1080,45 +1131,101 @@ std::string gpt_random_prompt(std::mt19937 & rng) {}//-// String parsing+// String utils//-std::string parse_samplers_input(std::string input) {- std::string output = "";+std::vector<std::string> string_split(std::string input, char separator) {+ std::vector<std::string> parts;+ size_t separator_pos = input.find(separator);+ while (separator_pos != std::string::npos) {+ std::string part = input.substr(0, separator_pos);+ parts.emplace_back(part);+ input = input.substr(separator_pos + 1);+ separator_pos = input.find(separator);+ }+ parts.emplace_back(input);+ return parts;+}++std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names) {+ std::unordered_map<std::string, llama_sampler_type> sampler_canonical_name_map {+ {"top_k", llama_sampler_type::TOP_K},+ {"top_p", llama_sampler_type::TOP_P},+ {"typical_p", llama_sampler_type::TYPICAL_P},+ {"min_p", llama_sampler_type::MIN_P},+ {"tfs_z", llama_sampler_type::TFS_Z},+ {"temperature", llama_sampler_type::TEMPERATURE}+ };+// since samplers names are written multiple ways// make it ready for both system names and input names- std::unordered_map<std::string, char> samplers_symbols {- {"top_k", 'k'},- {"top-k", 'k'},- {"top_p", 'p'},- {"top-p", 'p'},- {"nucleus", 'p'},- {"typical_p", 'y'},- {"typical-p", 'y'},- {"typical", 'y'},- {"min_p", 'm'},- {"min-p", 'm'},- {"tfs_z", 'f'},- {"tfs-z", 'f'},- {"tfs", 'f'},- {"temp", 't'},- {"temperature",'t'}+ std::unordered_map<std::string, llama_sampler_type> sampler_alt_name_map {+ {"top-k", llama_sampler_type::TOP_K},+ {"top-p", llama_sampler_type::TOP_P},+ {"nucleus", llama_sampler_type::TOP_P},+ {"typical-p", llama_sampler_type::TYPICAL_P},+ {"typical", llama_sampler_type::TYPICAL_P},+ {"min-p", llama_sampler_type::MIN_P},+ {"tfs-z", llama_sampler_type::TFS_Z},+ {"tfs", llama_sampler_type::TFS_Z},+ {"temp", llama_sampler_type::TEMPERATURE}};- // expected format example: "temp;top_k;tfs_z;typical_p;top_p;min_p"- size_t separator = input.find(';');- while (separator != input.npos) {- std::string name = input.substr(0,separator);- input = input.substr(separator+1);- separator = input.find(';');-- if (samplers_symbols.find(name) != samplers_symbols.end()) {- output += samplers_symbols[name];++ std::vector<llama_sampler_type> sampler_types;+ sampler_types.reserve(names.size());+ for (const auto & name : names)+ {+ auto sampler_item = sampler_canonical_name_map.find(name);+ if (sampler_item != sampler_canonical_name_map.end())+ {+ sampler_types.push_back(sampler_item->second);+ }+ else+ {+ if (allow_alt_names)+ {+ sampler_item = sampler_alt_name_map.find(name);+ if (sampler_item != sampler_alt_name_map.end())+ {+ sampler_types.push_back(sampler_item->second);+ }+ }}}- if (samplers_symbols.find(input) != samplers_symbols.end()) {- output += samplers_symbols[input];+ return sampler_types;+}++std::vector<llama_sampler_type> sampler_types_from_chars(const std::string & names_string) {+ std::unordered_map<char, llama_sampler_type> sampler_name_map {+ {'k', llama_sampler_type::TOP_K},+ {'p', llama_sampler_type::TOP_P},+ {'y', llama_sampler_type::TYPICAL_P},+ {'m', llama_sampler_type::MIN_P},+ {'f', llama_sampler_type::TFS_Z},+ {'t', llama_sampler_type::TEMPERATURE}+ };++ std::vector<llama_sampler_type> sampler_types;+ sampler_types.reserve(names_string.size());+ for (const auto & c : names_string) {+ const auto sampler_item = sampler_name_map.find(c);+ if (sampler_item != sampler_name_map.end()) {+ sampler_types.push_back(sampler_item->second);+ }+ }+ return sampler_types;+}++std::string sampler_type_to_name_string(llama_sampler_type sampler_type) {+ switch (sampler_type) {+ case llama_sampler_type::TOP_K: return "top_k";+ case llama_sampler_type::TFS_Z: return "tfs_z";+ case llama_sampler_type::TYPICAL_P: return "typical_p";+ case llama_sampler_type::TOP_P: return "top_p";+ case llama_sampler_type::MIN_P: return "min_p";+ case llama_sampler_type::TEMPERATURE: return "temperature";+ default : return "";}- return output;}//@@ -1520,9 +1627,10 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "cpu_has_avx512: %s\n", ggml_cpu_has_avx512() ? "true" : "false");fprintf(stream, "cpu_has_avx512_vbmi: %s\n", ggml_cpu_has_avx512_vbmi() ? "true" : "false");fprintf(stream, "cpu_has_avx512_vnni: %s\n", ggml_cpu_has_avx512_vnni() ? "true" : "false");- fprintf(stream, "cpu_has_blas: %s\n", ggml_cpu_has_blas() ? "true" : "false");fprintf(stream, "cpu_has_cublas: %s\n", ggml_cpu_has_cublas() ? "true" : "false");+ fprintf(stream, "cpu_has_vulkan: %s\n", ggml_cpu_has_vulkan() ? "true" : "false");fprintf(stream, "cpu_has_clblast: %s\n", ggml_cpu_has_clblast() ? "true" : "false");+ fprintf(stream, "cpu_has_kompute: %s\n", ggml_cpu_has_kompute() ? "true" : "false");fprintf(stream, "cpu_has_fma: %s\n", ggml_cpu_has_fma() ? "true" : "false");fprintf(stream, "cpu_has_gpublas: %s\n", ggml_cpu_has_gpublas() ? "true" : "false");fprintf(stream, "cpu_has_neon: %s\n", ggml_cpu_has_neon() ? "true" : "false");@@ -1532,6 +1640,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "cpu_has_blas: %s\n", ggml_cpu_has_blas() ? "true" : "false");fprintf(stream, "cpu_has_sse3: %s\n", ggml_cpu_has_sse3() ? "true" : "false");fprintf(stream, "cpu_has_vsx: %s\n", ggml_cpu_has_vsx() ? "true" : "false");+ fprintf(stream, "cpu_has_matmul_int8: %s\n", ggml_cpu_has_matmul_int8() ? "true" : "false");#ifdef NDEBUGfprintf(stream, "debug: false\n");@@ -1608,6 +1717,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l}fprintf(stream, "lora_base: %s\n", params.lora_base.c_str());fprintf(stream, "main_gpu: %d # default: 0\n", params.main_gpu);+ fprintf(stream, "min_keep: %d # default: 0 (disabled)\n", sparams.min_keep);fprintf(stream, "mirostat: %d # default: 0 (disabled)\n", sparams.mirostat);fprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau);fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta);@@ -1621,7 +1731,6 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false");fprintf(stream, "no_mul_mat_q: %s # default: false\n", !params.mul_mat_q ? "true" : "false");fprintf(stream, "no_penalize_nl: %s # default: false\n", !sparams.penalize_nl ? "true" : "false");- fprintf(stream, "numa: %s # default: false\n", params.numa ? "true" : "false");fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type);fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride);fprintf(stream, "presence_penalty: %f # default: 0.0\n", sparams.penalty_present);@@ -1646,16 +1755,16 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "rope_freq_base: %f # default: 10000.0\n", params.rope_freq_base);fprintf(stream, "rope_freq_scale: %f # default: 1.0\n", params.rope_freq_scale);- fprintf(stream, "seed: %d # default: -1 (random seed)\n", params.seed);+ fprintf(stream, "seed: %u # default: -1 (random seed)\n", params.seed);fprintf(stream, "simple_io: %s # default: false\n", params.simple_io ? "true" : "false");fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false");fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp);- const std::vector<float> tensor_split_vector(params.tensor_split, params.tensor_split + LLAMA_MAX_DEVICES);+ const std::vector<float> tensor_split_vector(params.tensor_split, params.tensor_split + llama_max_devices());dump_vector_float_yaml(stream, "tensor_split", tensor_split_vector);fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z);- fprintf(stream, "threads: %d # default: %d\n", params.n_threads, std::thread::hardware_concurrency());+ fprintf(stream, "threads: %d # default: %u\n", params.n_threads, std::thread::hardware_concurrency());fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k);fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p);fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p);@@ -1706,7 +1815,8 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) {if (cs_curr[j] < 0) { continue; }if (seqs.find(cs_curr[j]) == seqs.end()) {if (seqs.size() + 1 >= sizeof(slot_chars)) { break; }- seqs[cs_curr[j]] = seqs.size();+ const size_t sz = seqs.size();+ seqs[cs_curr[j]] = sz;}}if (seqs.size() + 1 >= sizeof(slot_chars)) { break; }diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 12803e1e..465a3ec6 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -45,50 +45,41 @@ extern char const *LLAMA_BUILD_TARGET;//int32_t get_num_physical_cores();-static inline int pick_default_thread_count() {- int n = get_num_physical_cores();- // last thing we want is to have 63 threads on AMD EPYC fighting to- // acquire a spinlock while the 64th thread finishes evaluating the- // last item of work in a segment of the model's computation graph.- if (n > 12) n = 12;- return n;-}-struct gpt_params {- uint32_t seed = -1; // RNG seed-- int32_t n_threads = pick_default_thread_count();- int32_t n_threads_draft = -1;- int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)- int32_t n_threads_batch_draft = -1;- int32_t n_predict = -1; // new tokens to predict- int32_t n_ctx = 512; // context size- int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)- int32_t n_keep = 0; // number of tokens to keep from initial prompt- int32_t n_draft = 8; // number of tokens to draft during speculative decoding- int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)- int32_t n_parallel = 1; // number of parallel sequences to decode- int32_t n_sequences = 1; // number of sequences to decode- float p_accept = 0.5f; // speculative decoding accept probability- float p_split = 0.1f; // speculative decoding split probability- int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default)- int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default)- llama_split_mode split_mode = LLAMA_SPLIT_LAYER; // how to split the model across GPUs- int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors- float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs- int32_t n_beams = 0; // if non-zero then use beam search of given width.- int32_t grp_attn_n = 1; // group-attention factor- int32_t grp_attn_w = 512; // group-attention width- int32_t n_print = -1; // print token count every n tokens (-1 = disabled)- float rope_freq_base = 0.0f; // RoPE base frequency- float rope_freq_scale = 0.0f; // RoPE frequency scaling factor- float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor- float yarn_attn_factor = 1.0f; // YaRN magnitude scaling factor- float yarn_beta_fast = 32.0f; // YaRN low correction dim- float yarn_beta_slow = 1.0f; // YaRN high correction dim- int32_t yarn_orig_ctx = 0; // YaRN original context length- int8_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; // TODO: better to be int32_t for alignment- // pinging @cebtenzzre+ uint32_t seed = -1; // RNG seed++ int32_t n_threads = get_num_physical_cores();+ int32_t n_threads_draft = -1;+ int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)+ int32_t n_threads_batch_draft = -1;+ int32_t n_predict = -1; // new tokens to predict+ int32_t n_ctx = 512; // context size+ int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)+ int32_t n_keep = 0; // number of tokens to keep from initial prompt+ int32_t n_draft = 8; // number of tokens to draft during speculative decoding+ int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)+ int32_t n_parallel = 1; // number of parallel sequences to decode+ int32_t n_sequences = 1; // number of sequences to decode+ float p_accept = 0.5f; // speculative decoding accept probability+ float p_split = 0.1f; // speculative decoding split probability+ int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default)+ int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default)+ llama_split_mode split_mode = LLAMA_SPLIT_LAYER; // how to split the model across GPUs+ int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors+ float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs+ int32_t n_beams = 0; // if non-zero then use beam search of given width.+ int32_t grp_attn_n = 1; // group-attention factor+ int32_t grp_attn_w = 512; // group-attention width+ int32_t n_print = -1; // print token count every n tokens (-1 = disabled)+ float rope_freq_base = 0.0f; // RoPE base frequency+ float rope_freq_scale = 0.0f; // RoPE frequency scaling factor+ float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor+ float yarn_attn_factor = 1.0f; // YaRN magnitude scaling factor+ float yarn_beta_fast = 32.0f; // YaRN low correction dim+ float yarn_beta_slow = 1.0f; // YaRN high correction dim+ int32_t yarn_orig_ctx = 0; // YaRN original context length+ int32_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED;+ ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED;// // sampling parametersstruct llama_sampling_params sparams;@@ -147,14 +138,12 @@ struct gpt_params {bool logits_all = false; // return logits for all tokens in the batchbool use_mmap = true; // use mmap for faster loadsbool use_mlock = false; // use mlock to keep model in memory- bool numa = false; // attempt optimizations that help on some NUMA systemsbool verbose_prompt = false; // print prompt tokens before generationbool display_prompt = true; // print prompt before generationbool infill = false; // use infill modebool dump_kv_cache = false; // dump the KV cache contents for debugging purposesbool no_kv_offload = false; // disable KV offloading- bool unsecure = false; // disable pledge() sandboxingstd::string cache_type_k = "f16"; // KV cache data type for the Kstd::string cache_type_v = "f16"; // KV cache data type for the V@@ -176,10 +165,13 @@ std::string gpt_random_prompt(std::mt19937 & rng);void process_escapes(std::string& input);//-// String parsing+// String utils//-std::string parse_samplers_input(std::string input);+std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);+std::vector<llama_sampler_type> sampler_types_from_chars(const std::string & names_string);+std::vector<std::string> string_split(std::string input, char separator);+std::string sampler_type_to_name_string(llama_sampler_type sampler_type);//// Model utilsdiff --git a/llama.cpp/ggml-alloc.c b/llama.cpp/ggml-alloc.cindex b036d975..cbffba8a 100644--- a/llama.cpp/ggml-alloc.c+++ b/llama.cpp/ggml-alloc.c@@ -11,6 +11,10 @@#include <stdlib.h>#include <string.h>+#ifndef NDEBUG+#define NDEBUG // [jart] delete printf debugging+#endif+#define MAX(a, b) ((a) > (b) ? (a) : (b))#define MAX_FREE_BLOCKS 256@@ -19,6 +23,50 @@//#define AT_PRINTF(...) fprintf(stderr, __VA_ARGS__)#define AT_PRINTF(...)++static bool ggml_is_view(const struct ggml_tensor * t) {+ return t->view_src != NULL;+}++static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {+ if (a->type != b->type) {+ return false;+ }+ for (int i = 0; i < GGML_MAX_DIMS; i++) {+ if (a->ne[i] != b->ne[i]) {+ return false;+ }+ if (a->nb[i] != b->nb[i]) {+ return false;+ }+ }+ return true;+}++static bool ggml_op_can_inplace(enum ggml_op op) {+ switch (op) {+ case GGML_OP_SCALE:+ case GGML_OP_DIAG_MASK_ZERO:+ case GGML_OP_DIAG_MASK_INF:+ case GGML_OP_ADD:+ case GGML_OP_ADD1:+ case GGML_OP_SUB:+ case GGML_OP_MUL:+ case GGML_OP_DIV:+ case GGML_OP_SQR:+ case GGML_OP_SQRT:+ case GGML_OP_LOG:+ case GGML_OP_UNARY:+ case GGML_OP_ROPE:+ case GGML_OP_RMS_NORM:+ case GGML_OP_SOFT_MAX:+ return true;++ default:+ return false;+ }+}+// TODO: GGML_PAD ?static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {assert(alignment && !(alignment & (alignment - 1))); // power of 2@@ -26,66 +74,102 @@ static size_t aligned_offset(const void * buffer, size_t offset, size_t alignmenreturn offset + align;}+// tallocr+struct ggml_tallocr {+ ggml_backend_buffer_t buffer;+ void * base;+ size_t alignment;+ size_t offset;+};++ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer) {+ ggml_tallocr_t talloc = malloc(sizeof(struct ggml_tallocr));+ if (talloc == NULL) {+ return NULL;+ }++ void * base = ggml_backend_buffer_get_base(buffer);+ size_t align = ggml_backend_buffer_get_alignment(buffer);++ assert(align && !(align & (align - 1))); // power of 2++ *talloc = (struct ggml_tallocr) {+ /*.buffer = */ buffer,+ /*.base = */ base,+ /*.alignment = */ align,+ /*.offset = */ aligned_offset(base, 0, align),+ };+ return talloc;+}++void ggml_tallocr_free(ggml_tallocr_t talloc) {+ free(talloc);+}++void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor) {+ size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor);+ size = GGML_PAD(size, talloc->alignment);++ if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) {+ fprintf(stderr, "%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\n",+ __func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset);+ GGML_ASSERT(!"not enough space in the buffer");+ return;+ }++ void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset;+ talloc->offset += size;++ assert(((uintptr_t)addr % talloc->alignment) == 0);++ ggml_backend_tensor_alloc(talloc->buffer, tensor, addr);+}++// dynamic tensor allocator+struct free_block {- void * addr;+ size_t offset;size_t size;};-struct ggml_tallocr {- struct ggml_backend_buffer * buffer;- bool buffer_owned;- void * base;+struct ggml_dyn_tallocr {size_t alignment;-int n_free_blocks;struct free_block free_blocks[MAX_FREE_BLOCKS];-size_t max_size;- bool measure;-#ifdef GGML_ALLOCATOR_DEBUG- struct ggml_tensor * allocated_tensors[1024];+ struct {+ const struct ggml_tensor * tensor;+ size_t offset;+ } allocated_tensors[1024];#endif};#ifdef GGML_ALLOCATOR_DEBUG-static void add_allocated_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {+static void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {for (int i = 0; i < 1024; i++) {- if (alloc->allocated_tensors[i] == NULL) {- alloc->allocated_tensors[i] = tensor;+ if (alloc->allocated_tensors[i].tensor == NULL) {+ alloc->allocated_tensors[i].tensor = tensor;+ alloc->allocated_tensors[i].offset = offset;return;}}GGML_ASSERT(!"out of allocated_tensors");}-static void remove_allocated_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {+static void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {for (int i = 0; i < 1024; i++) {- if (alloc->allocated_tensors[i] == tensor ||- (alloc->allocated_tensors[i] != NULL && alloc->allocated_tensors[i]->data == tensor->data)) {- alloc->allocated_tensors[i] = NULL;+ if (alloc->allocated_tensors[i].offset == offset) {+ alloc->allocated_tensors[i].tensor = NULL;return;}}- printf("tried to free tensor %s not found\n", tensor->name);+ fprintf(stderr, "tried to free tensor %s not found\n", tensor->name);GGML_ASSERT(!"tensor not found");}#endif-// check if a tensor is allocated by this buffer-static bool ggml_tallocr_is_own(ggml_tallocr_t alloc, const struct ggml_tensor * tensor) {- return tensor->buffer == alloc->buffer && (!tensor->view_src || tensor->view_src->buffer == alloc->buffer);-}--static bool ggml_is_view(struct ggml_tensor * t) {- return t->view_src != NULL;-}--void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {- GGML_ASSERT(!ggml_is_view(tensor)); // views generally get data pointer from one of their sources- GGML_ASSERT(tensor->data == NULL); // avoid allocating tensor which already has memory allocated-- size_t size = ggml_backend_buffer_get_alloc_size(alloc->buffer, tensor);+static size_t ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) {size = aligned_offset(NULL, size, alloc->alignment);AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size);@@ -111,16 +195,17 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {if (block->size >= size) {best_fit_block = alloc->n_free_blocks - 1;} else {- fprintf(stderr, "%s: not enough space in the buffer to allocate %s (needed %zu, largest block available %zu)\n",- __func__, tensor->name, size, max_avail);+ // this should never happen+ fprintf(stderr, "%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\n",+ __func__, size, max_avail);GGML_ASSERT(!"not enough space in the buffer");- return;+ GGML_UNREACHABLE();}}struct free_block * block = &alloc->free_blocks[best_fit_block];- void * addr = block->addr;- block->addr = (char*)block->addr + size;+ size_t offset = block->offset;+ block->offset = offset + size;block->size -= size;if (block->size == 0) {// remove block if empty@@ -130,59 +215,63 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {}}- AT_PRINTF("block %d, addr %p\n", best_fit_block, addr);-- tensor->data = addr;- tensor->buffer = alloc->buffer;- if (!alloc->measure) {- ggml_backend_buffer_init_tensor(alloc->buffer, tensor);- }+ AT_PRINTF("block %d, offset %zu\n", best_fit_block, offset);#ifdef GGML_ALLOCATOR_DEBUG- add_allocated_tensor(alloc, tensor);- size_t cur_max = (char*)addr - (char*)alloc->base + size;+ add_allocated_tensor(alloc, offset, tensor);+ size_t cur_max = offset + size;if (cur_max > alloc->max_size) {- printf("max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0);+ // sort allocated_tensors by offset+ for (int i = 0; i < 1024; i++) {+ for (int j = i + 1; j < 1024; j++) {+ if (alloc->allocated_tensors[i].offset > alloc->allocated_tensors[j].offset) {+ const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor;+ size_t tmp_offset = alloc->allocated_tensors[i].offset;+ alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor;+ alloc->allocated_tensors[i].offset = alloc->allocated_tensors[j].offset;+ alloc->allocated_tensors[j].tensor = tmp_tensor;+ alloc->allocated_tensors[j].offset = tmp_offset;+ }+ }+ }+ fprintf(stderr, "max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0);for (int i = 0; i < 1024; i++) {- if (alloc->allocated_tensors[i]) {- printf("%s (%.2f MB) ", alloc->allocated_tensors[i]->name, ggml_nbytes(alloc->allocated_tensors[i]) / 1024.0 / 1024.0);+ if (alloc->allocated_tensors[i].tensor) {+ fprintf(stderr, "%s [%zx-%zx] (%.2f MB) ", alloc->allocated_tensors[i].tensor->name,+ alloc->allocated_tensors[i].offset,+ alloc->allocated_tensors[i].offset + ggml_nbytes(alloc->allocated_tensors[i].tensor),+ ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0);}}- printf("\n");+ fprintf(stderr, "\n");}#endif- alloc->max_size = MAX(alloc->max_size, (char*)addr - (char*)alloc->base + size);-}+ alloc->max_size = MAX(alloc->max_size, offset + size);-// this is a very naive implementation, but for our case the number of free blocks should be very small-static void ggml_tallocr_free_tensor(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {- if (ggml_tallocr_is_own(alloc, tensor) == false) {- // the tensor was not allocated in this buffer- // this can happen because the graph allocator will try to free weights and other tensors from different buffers- // the easiest way to deal with this is just to ignore it- // AT_PRINTF("ignoring %s (their buffer: %p, our buffer: %p)\n", tensor->name, (void *)tensor->buffer, (void *)alloc->buffer);- return;- }+ return offset;- void * ptr = tensor->data;+ GGML_UNUSED(tensor);+}- size_t size = ggml_backend_buffer_get_alloc_size(alloc->buffer, tensor);+// this is a very naive implementation, but for our case the number of free blocks should be very small+static void ggml_dyn_tallocr_free_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, size_t size, const struct ggml_tensor * tensor) {size = aligned_offset(NULL, size, alloc->alignment);- AT_PRINTF("%s: freeing %s at %p (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, ptr, size, alloc->n_free_blocks);++ AT_PRINTF("%s: freeing %s at %zu (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, offset, size, alloc->n_free_blocks);#ifdef GGML_ALLOCATOR_DEBUG- remove_allocated_tensor(alloc, tensor);+ remove_allocated_tensor(alloc, offset, tensor);#endif// see if we can merge with an existing blockfor (int i = 0; i < alloc->n_free_blocks; i++) {struct free_block * block = &alloc->free_blocks[i];// check if ptr is at the end of the block- if ((char*)block->addr + block->size == ptr) {+ if (block->offset + block->size == offset) {block->size += size;// check if we can merge with the next block- if (i < alloc->n_free_blocks - 1 && (char*)block->addr + block->size == alloc->free_blocks[i+1].addr) {+ if (i < alloc->n_free_blocks - 1 && block->offset + block->size == alloc->free_blocks[i+1].offset) {block->size += alloc->free_blocks[i+1].size;alloc->n_free_blocks--;for (int j = i+1; j < alloc->n_free_blocks; j++) {@@ -192,11 +281,11 @@ static void ggml_tallocr_free_tensor(ggml_tallocr_t alloc, struct ggml_tensor *return;}// check if ptr is at the beginning of the block- if ((char*)ptr + size == block->addr) {- block->addr = ptr;+ if (offset + size == block->offset) {+ block->offset = offset;block->size += size;// check if we can merge with the previous block- if (i > 0 && (char*)alloc->free_blocks[i-1].addr + alloc->free_blocks[i-1].size == block->addr) {+ if (i > 0 && alloc->free_blocks[i-1].offset + alloc->free_blocks[i-1].size == block->offset) {alloc->free_blocks[i-1].size += block->size;alloc->n_free_blocks--;for (int j = i; j < alloc->n_free_blocks; j++) {@@ -210,7 +299,7 @@ static void ggml_tallocr_free_tensor(ggml_tallocr_t alloc, struct ggml_tensor *GGML_ASSERT(alloc->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks");// insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster)int insert_pos = 0;- while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].addr < ptr) {+ while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].offset < offset) {insert_pos++;}// shift all blocks from insert_pos onward to make room for the new block@@ -218,608 +307,616 @@ static void ggml_tallocr_free_tensor(ggml_tallocr_t alloc, struct ggml_tensor *alloc->free_blocks[i] = alloc->free_blocks[i-1];}// insert the new block- alloc->free_blocks[insert_pos].addr = ptr;+ alloc->free_blocks[insert_pos].offset = offset;alloc->free_blocks[insert_pos].size = size;alloc->n_free_blocks++;++ GGML_UNUSED(tensor);}-void ggml_tallocr_reset(ggml_tallocr_t alloc) {+static void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) {alloc->n_free_blocks = 1;- size_t align_offset = aligned_offset(alloc->base, 0, alloc->alignment);- alloc->free_blocks[0].addr = (char *)alloc->base + align_offset;-- if (alloc->measure) {- alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows- } else {- alloc->free_blocks[0].size = ggml_backend_buffer_get_size(alloc->buffer) - align_offset;- ggml_backend_buffer_reset(alloc->buffer);- }+ alloc->free_blocks[0].offset = 0;+ alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows+ alloc->max_size = 0;}-ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment) {- struct ggml_backend_buffer * buffer = ggml_backend_cpu_buffer_from_ptr(data, size);-- ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr));+static struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment) {+ struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr));- *alloc = (struct ggml_tallocr) {- /*.buffer = */ buffer,- /*.buffer_owned = */ true,- /*.base = */ ggml_backend_buffer_get_base(buffer),+ *alloc = (struct ggml_dyn_tallocr) {/*.alignment = */ alignment,/*.n_free_blocks = */ 0,/*.free_blocks = */ {{0}},/*.max_size = */ 0,- /*.measure = */ false,#ifdef GGML_ALLOCATOR_DEBUG- /*.allocated_tensors = */ {0},+ /*.allocated_tensors = */ {{0}},#endif};- ggml_tallocr_reset(alloc);+ ggml_dyn_tallocr_reset(alloc);return alloc;}-ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment) {- ggml_tallocr_t alloc = ggml_tallocr_new((void *)0x1000, SIZE_MAX/2, alignment);- alloc->measure = true;-- return alloc;-}--ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft) {- // create a backend buffer to get the correct tensor allocation sizes- ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, 1);-- // TODO: move alloc initialization to a common ggml_tallocr_new_impl function- ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer);- alloc->buffer_owned = true;- alloc->measure = true;- ggml_tallocr_reset(alloc);- return alloc;+static void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) {+ free(alloc);}-ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend) {- return ggml_tallocr_new_measure_from_buft(ggml_backend_get_default_buffer_type(backend));+static size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc) {+ return alloc->max_size;}-ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size) {- // create a backend buffer to get the correct tensor allocation sizes- ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);- ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer);- alloc->buffer_owned = true;- return alloc;-}-ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size) {- return ggml_tallocr_new_from_buft(ggml_backend_get_default_buffer_type(backend), size);-}+/////////////////////////////////////-ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer) {- ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr));+// graph allocator- *alloc = (struct ggml_tallocr) {- /*.buffer = */ buffer,- /*.buffer_owned = */ false,- /*.base = */ ggml_backend_buffer_get_base(buffer),- /*.alignment = */ ggml_backend_buffer_get_alignment(buffer),- /*.n_free_blocks = */ 0,- /*.free_blocks = */ {{0}},- /*.max_size = */ 0,- /*.measure = */ false,-#ifdef GGML_ALLOCATOR_DEBUG- /*.allocated_tensors = */ {0},-#endif- };+struct hash_node {+ int n_children;+ int n_views;+ int buffer_id;+ size_t offset; // offset within the buffer+ bool allocated;+};- ggml_tallocr_reset(alloc);+//+struct tensor_alloc {+ size_t offset;+ size_t size_max; // 0 = pre-allocated, unused, or view+};- return alloc;-}+struct node_alloc {+ int buffer_id;+ struct tensor_alloc dst;+ struct tensor_alloc src[GGML_MAX_SRC];+};-struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t alloc) {- return alloc->buffer;-}+struct ggml_gallocr {+ ggml_backend_buffer_type_t * bufts; // [n_buffers]+ ggml_backend_buffer_t * buffers; // [n_buffers]+ struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers]+ int n_buffers;-void ggml_tallocr_free(ggml_tallocr_t alloc) {- if (alloc == NULL) {- return;- }+ struct ggml_hash_set hash_set;+ struct hash_node * hash_values; // [hash_set.size]- if (alloc->buffer_owned) {- ggml_backend_buffer_free(alloc->buffer);- }- free(alloc);-}+ struct node_alloc * node_allocs; // [n_nodes]+ int n_nodes;-bool ggml_tallocr_is_measure(ggml_tallocr_t alloc) {- return alloc->measure;-}+ struct tensor_alloc * leaf_allocs; // [n_leafs]+ int n_leafs;+};-size_t ggml_tallocr_max_size(ggml_tallocr_t alloc) {- // FIXME: changes in the tensor sizes compared to the measure graph may cause allocations to fail- // to avoid this, we add a 10% margin to the buffer size- return alloc->max_size + alloc->max_size/10;-}+ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) {+ ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(sizeof(struct ggml_gallocr), 1);+ GGML_ASSERT(galloc != NULL);-// graph allocator+ galloc->bufts = calloc(sizeof(ggml_backend_buffer_type_t) * n_bufs, 1);+ GGML_ASSERT(galloc->bufts != NULL);-struct hash_node {- int n_children;- int n_views;-};+ galloc->buffers = calloc(sizeof(ggml_backend_buffer_t) * n_bufs, 1);+ GGML_ASSERT(galloc->buffers != NULL);-struct ggml_gallocr {- ggml_tallocr_t talloc;- struct ggml_hash_set hash_set;- struct hash_node * hash_values;- size_t hash_values_size;- ggml_tallocr_t * hash_allocs;- int * parse_seq;- int parse_seq_len;-};+ galloc->buf_tallocs = calloc(sizeof(struct ggml_dyn_tallocr *) * n_bufs, 1);+ GGML_ASSERT(galloc->buf_tallocs != NULL);-ggml_gallocr_t ggml_gallocr_new(void) {- ggml_gallocr_t galloc = (ggml_gallocr_t)malloc(sizeof(struct ggml_gallocr));-- *galloc = (struct ggml_gallocr) {- /*.talloc = */ NULL,- /*.hash_set = */ {0},- /*.hash_values = */ NULL,- /*.hash_values_size = */ 0,- /*.hash_allocs = */ NULL,- /*.parse_seq = */ NULL,- /*.parse_seq_len = */ 0,- };+ for (int i = 0; i < n_bufs; i++) {+ galloc->bufts[i] = bufts[i];+ galloc->buffers[i] = NULL;+ size_t alignment = ggml_backend_buft_get_alignment(bufts[i]);+ galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment);+ }+ galloc->n_buffers = n_bufs;return galloc;}+ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) {+ return ggml_gallocr_new_n(&buft, 1);+}+void ggml_gallocr_free(ggml_gallocr_t galloc) {if (galloc == NULL) {return;}- if (galloc->hash_set.keys != NULL) {- free(galloc->hash_set.keys);- }- if (galloc->hash_values != NULL) {- free(galloc->hash_values);- }- if (galloc->hash_allocs != NULL) {- free(galloc->hash_allocs);- }- if (galloc->parse_seq != NULL) {- free(galloc->parse_seq);+ for (int i = 0; i < galloc->n_buffers; i++) {+ if (galloc->buffers != NULL) {+ ggml_backend_buffer_free(galloc->buffers[i]);+ }+ if (galloc->buf_tallocs != NULL) {+ ggml_dyn_tallocr_free(galloc->buf_tallocs[i]);+ }}++ free(galloc->hash_set.keys);+ free(galloc->hash_values);+ free(galloc->bufts);+ free(galloc->buffers);+ free(galloc->buf_tallocs);+ free(galloc->node_allocs);+ free(galloc->leaf_allocs);free(galloc);}-void ggml_gallocr_set_parse_seq(ggml_gallocr_t galloc, const int * list, int n) {- free(galloc->parse_seq);- galloc->parse_seq = malloc(sizeof(int) * n);-- for (int i = 0; i < n; i++) {- galloc->parse_seq[i] = list[i];- }- galloc->parse_seq_len = n;-}+typedef struct ggml_gallocr * ggml_gallocr_t;-static struct hash_node * hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {+static struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {size_t i = ggml_hash_find_or_insert(galloc->hash_set, t);return &galloc->hash_values[i];}-static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {- if (a->type != b->type) {- return false;- }- for (int i = 0; i < GGML_MAX_DIMS; i++) {- if (a->ne[i] != b->ne[i]) {- return false;- }- if (a->nb[i] != b->nb[i]) {- return false;- }- }- return true;+static bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) {+ return ggml_gallocr_hash_get(galloc, t)->allocated;}-static bool ggml_op_can_inplace(enum ggml_op op) {- switch (op) {- case GGML_OP_SCALE:- case GGML_OP_DIAG_MASK_ZERO:- case GGML_OP_DIAG_MASK_INF:- case GGML_OP_ADD:- case GGML_OP_ADD1:- case GGML_OP_SUB:- case GGML_OP_MUL:- case GGML_OP_DIV:- case GGML_OP_SQR:- case GGML_OP_SQRT:- case GGML_OP_LOG:- case GGML_OP_UNARY:- case GGML_OP_ROPE:- case GGML_OP_RMS_NORM:- case GGML_OP_SOFT_MAX:- return true;-- default:- return false;- }+static void ggml_gallocr_set_node_offset(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, size_t offset) {+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);+ hn->buffer_id = buffer_id;+ hn->offset = offset;+ hn->allocated = true;}-static ggml_tallocr_t node_tallocr(ggml_gallocr_t galloc, struct ggml_tensor * node) {- if (galloc->talloc != NULL) {- return galloc->talloc;- }-- return galloc->hash_allocs[ggml_hash_find_or_insert(galloc->hash_set, node)];+static bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) {+ return t->data != NULL || ggml_gallocr_hash_get(galloc, t)->allocated;}-static void init_view(ggml_gallocr_t galloc, struct ggml_tensor * view, bool update_backend) {- ggml_tallocr_t alloc = node_tallocr(galloc, view);-- GGML_ASSERT(view->view_src != NULL && view->view_src->data != NULL);- if (update_backend) {- view->backend = view->view_src->backend;- }- // views are initialized in the alloc buffer rather than the view_src buffer- view->buffer = alloc->buffer;- view->data = (char *)view->view_src->data + view->view_offs;+static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) {+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);- assert(ggml_tallocr_is_measure(alloc) || !view->buffer || view->buffer->buft == alloc->buffer->buft);+ if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_is_view(node)) {+ hn->allocated = true;+ assert(hn->offset == 0);- if (!alloc->measure) {- ggml_backend_buffer_init_tensor(alloc->buffer, view);- }-}+ // try to reuse a parent's buffer (inplace)+ if (ggml_op_can_inplace(node->op)) {+ for (int i = 0; i < GGML_MAX_SRC; i++) {+ struct ggml_tensor * parent = node->src[i];+ if (parent == NULL) {+ continue;+ }-static void allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {- ggml_tallocr_t alloc = node_tallocr(galloc, node);+ // if the node's data is external, then we cannot re-use it+ if (!ggml_gallocr_is_own(galloc, parent)) {+ AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);+ continue;+ }- if (node->data == NULL) {- if (ggml_is_view(node)) {- init_view(galloc, node, true);- } else {- // see if we can reuse a parent's buffer (inplace)- if (ggml_op_can_inplace(node->op)) {- for (int i = 0; i < GGML_MAX_SRC; i++) {- struct ggml_tensor * parent = node->src[i];- if (parent == NULL) {- break;- }+ // outputs cannot be reused+ if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) {+ AT_PRINTF("not reusing parent %s for %s as it is an output\n", parent->name, node->name);+ continue;+ }- // if the node's data is external, then we cannot re-use it- if (ggml_tallocr_is_own(alloc, parent) == false) {- AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);- continue;- }+ if (!ggml_are_same_layout(node, parent)) {+ AT_PRINTF("not reusing parent %s for %s as layouts are different\n", parent->name, node->name);+ continue;+ }- struct hash_node * p_hn = hash_get(galloc, parent);- if (parent->data != NULL && p_hn->n_children == 1 && p_hn->n_views == 0 && ggml_are_same_layout(node, parent)) {- if (ggml_is_view(parent)) {- struct ggml_tensor * view_src = parent->view_src;- struct hash_node * view_src_hn = hash_get(galloc, view_src);- if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {- // TODO: the offset of the view parent must be kept to ensure that the op doesn't overwrite- // the parent's data that it will need later (same layout requirement). the problem is that then- // we cannot free the tensor because the original address of the allocation is lost.- // adding a view_src pointer to the tensor would solve this and simplify the code dealing with views- // for now, we only reuse the parent's data if the offset is zero (view_src->data == parent->data)- AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);- node->view_src = view_src;- view_src_hn->n_views += 1;- init_view(galloc, node, false);- return;- }- } else {- AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);- node->view_src = parent;- p_hn->n_views += 1;- init_view(galloc, node, false);+ struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);+ if (p_hn->n_children == 1 && p_hn->n_views == 0) {+ if (ggml_is_view(parent)) {+ struct ggml_tensor * view_src = parent->view_src;+ struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);+ if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {+ AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);+ assert(view_src_hn->offset == p_hn->offset);+ hn->buffer_id = p_hn->buffer_id;+ hn->offset = p_hn->offset;+ p_hn->allocated = false; // avoid freeing the parent+ view_src_hn->allocated = false;return;}+ } else {+ AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);+ hn->buffer_id = p_hn->buffer_id;+ hn->offset = p_hn->offset;+ p_hn->allocated = false; // avoid freeing the parent+ return;}}}- ggml_tallocr_alloc(alloc, node);}+ // allocate tensor from the buffer+ struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];+ ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];+ size_t size = ggml_backend_buft_get_alloc_size(buft, node);+ size_t offset = ggml_dyn_tallocr_alloc(alloc, size, node);+ hn->buffer_id = buffer_id;+ hn->offset = offset;+ return;}}-static void free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {- ggml_tallocr_t alloc = node_tallocr(galloc, node);+static void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) {+ // graph outputs are never freed+ if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {+ AT_PRINTF("not freeing output %s\n", node->name);+ return;+ }++ struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];+ ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);+ size_t offset = hn->offset;+ size_t size = ggml_backend_buft_get_alloc_size(buft, node);+ ggml_dyn_tallocr_free_tensor(alloc, offset, size, node);+ hn->allocated = false;+}- ggml_tallocr_free_tensor(alloc, node);+static int get_node_buffer_id(const int * node_buffer_ids, int i) {+ return node_buffer_ids ? node_buffer_ids[i] : 0;}-static void ggml_tallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * gf) {- const int * parse_seq = galloc->parse_seq;- int parse_seq_len = galloc->parse_seq_len;+static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) {+ // clear hash tables+ memset(galloc->hash_set.keys, 0, galloc->hash_set.size * sizeof(struct ggml_tensor *));+ memset(galloc->hash_values, 0, galloc->hash_set.size * sizeof(struct hash_node));// count number of children and views- for (int i = 0; i < gf->n_nodes; i++) {- struct ggml_tensor * node = gf->nodes[i];+ // allocate all graph inputs and leafs first to avoid overwriting them+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view(node)) {struct ggml_tensor * view_src = node->view_src;- hash_get(galloc, view_src)->n_views += 1;- if (node->buffer == NULL && node->data != NULL) {- // view of a pre-allocated tensor, didn't call init_view() yet- init_view(galloc, node, true);- }+ ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;+ }++ if (node->flags & GGML_TENSOR_FLAG_INPUT) {+ ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i));}for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * parent = node->src[j];- if (parent == NULL) {- break;+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ continue;}- hash_get(galloc, parent)->n_children += 1;- if (ggml_is_view(parent) && parent->buffer == NULL && parent->data != NULL) {- init_view(galloc, parent, true);++ ggml_gallocr_hash_get(galloc, src)->n_children += 1;++ // allocate explicit inputs and leafs+ if (src->flags & GGML_TENSOR_FLAG_INPUT || src->op == GGML_OP_NONE) {+ ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i));}}- }+ }++ // allocate the remaining leafs that are unused on the graph+ // these are effectively static tensors that the application is not using in the graph, but may still want to allocate for other purposes+ for (int i = 0; i < graph->n_leafs; i++) {+ struct ggml_tensor * leaf = graph->leafs[i];+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);++ if (hn->n_children == 0) {+ assert(!hn->allocated);+ // since buffer ids are only given for nodes, these leafs are always allocated in the first buffer+ ggml_gallocr_allocate_node(galloc, leaf, 0);+ }+ }// allocate tensors- // if we have parse_seq then we allocate nodes following the list, and we only free nodes at barriers- int last_barrier_pos = 0;- int n_nodes = parse_seq_len ? parse_seq_len : gf->n_nodes;-- for (int ind = 0; ind < n_nodes; ind++) {- // allocate a node if there is no parse_seq or this is not a barrier- if (parse_seq_len == 0 || parse_seq[ind] != -1) {- int i = parse_seq_len ? parse_seq[ind] : ind;- struct ggml_tensor * node = gf->nodes[i];-- // allocate parents (leafs)- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * parent = node->src[j];- if (parent == NULL) {- break;- }- allocate_node(galloc, parent);+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ int buffer_id = get_node_buffer_id(node_buffer_ids, i);++ // allocate parents (only leafs need to be allocated at this point)+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * parent = node->src[j];+ if (parent == NULL) {+ continue;}+ ggml_gallocr_allocate_node(galloc, parent, buffer_id);+ }- // allocate node- allocate_node(galloc, node);+ // allocate node+ ggml_gallocr_allocate_node(galloc, node, buffer_id);- AT_PRINTF("exec: %s (%s) <= ", ggml_op_name(node->op), node->name);- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * parent = node->src[j];- if (parent == NULL) {- break;- }- AT_PRINTF("%s", parent->name);- if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {- AT_PRINTF(", ");- }+ AT_PRINTF("exec: %s (%s) <= ", ggml_op_desc(node), node->name);+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * parent = node->src[j];+ if (parent == NULL) {+ continue;+ }+ AT_PRINTF("%s", parent->name);+ if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {+ AT_PRINTF(", ");}- AT_PRINTF("\n");}+ AT_PRINTF("\n");// update parents- // update immediately if there is no parse_seq- // update only at barriers if there is parse_seq- if ((parse_seq_len == 0) || parse_seq[ind] == -1) {- int update_start = parse_seq_len ? last_barrier_pos : ind;- int update_end = parse_seq_len ? ind : ind + 1;- for (int i = update_start; i < update_end; i++) {- int node_i = parse_seq_len ? parse_seq[i] : i;- struct ggml_tensor * node = gf->nodes[node_i];-- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * parent = node->src[j];- if (parent == NULL) {- break;- }- struct hash_node * p_hn = hash_get(galloc, parent);- p_hn->n_children -= 1;-- //AT_PRINTF("parent %s: %d children, %d views\n", parent->name, parent->n_children, parent->n_views);-- if (p_hn->n_children == 0 && p_hn->n_views == 0) {- if (ggml_is_view(parent)) {- struct ggml_tensor * view_src = parent->view_src;- struct hash_node * view_src_hn = hash_get(galloc, view_src);- view_src_hn->n_views -= 1;- AT_PRINTF("view_src %s: %d children, %d views\n", view_src->name, view_src_hn->n_children, view_src_hn->n_views);- if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0) {- free_node(galloc, view_src);- }- }- else {- free_node(galloc, parent);- }+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * parent = node->src[j];+ if (parent == NULL) {+ continue;+ }+ struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);+ p_hn->n_children -= 1;++ AT_PRINTF("parent %s: %d children, %d views, allocated: %d\n",+ parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated);++ if (p_hn->n_children == 0 && p_hn->n_views == 0) {+ if (ggml_is_view(parent)) {+ struct ggml_tensor * view_src = parent->view_src;+ struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);+ view_src_hn->n_views -= 1;+ AT_PRINTF("view_src %s: %d children, %d views\n",+ view_src->name, view_src_hn->n_children, view_src_hn->n_views);+ if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) {+ ggml_gallocr_free_node(galloc, view_src, buffer_id);}}+ else if (p_hn->allocated) {+ ggml_gallocr_free_node(galloc, parent, buffer_id);+ }}AT_PRINTF("\n");- if (parse_seq_len) {- last_barrier_pos = ind + 1;- }}}}-size_t ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, ggml_tallocr_t talloc, struct ggml_cgraph * graph) {+bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) {size_t hash_size = graph->visited_hash_table.size;- // check if the hash table is initialized and large enough+ // initialize hash tableif (galloc->hash_set.size < hash_size) {- if (galloc->hash_set.keys != NULL) {- free(galloc->hash_set.keys);- }- if (galloc->hash_values != NULL) {- free(galloc->hash_values);- }- galloc->hash_set.keys = malloc(sizeof(struct ggml_tensor *) * hash_size);+ free(galloc->hash_set.keys);+ free(galloc->hash_values);galloc->hash_set.size = hash_size;- galloc->hash_values = malloc(sizeof(struct hash_node) * hash_size);+ galloc->hash_set.keys = calloc(sizeof(struct ggml_tensor *), hash_size);+ galloc->hash_values = calloc(sizeof(struct hash_node), hash_size);+ GGML_ASSERT(galloc->hash_set.keys != NULL);+ GGML_ASSERT(galloc->hash_values != NULL);+ } else {+ // reset hash table+ memset(galloc->hash_set.keys, 0, sizeof(struct ggml_tensor *) * galloc->hash_set.size);+ memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size);}- // reset hash table- memset(galloc->hash_set.keys, 0, sizeof(struct ggml_tensor *) * hash_size);- memset(galloc->hash_values, 0, sizeof(struct hash_node) * hash_size);-- galloc->talloc = talloc;- ggml_tallocr_alloc_graph_impl(galloc, graph);- galloc->talloc = NULL;-- size_t max_size = ggml_tallocr_max_size(talloc);-- return max_size;-}--void ggml_gallocr_alloc_graph_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, struct ggml_hash_set hash_set, ggml_tallocr_t * hash_node_talloc) {- const size_t hash_size = hash_set.size;-- GGML_ASSERT(hash_size >= (size_t)(graph->n_nodes + graph->n_leafs));+ // reset allocators+ for (int i = 0; i < galloc->n_buffers; i++) {+ ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]);+ }- galloc->talloc = NULL;+ // allocate in hash table+ ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids);- // alloc hash_values if needed- if (galloc->hash_values == NULL || galloc->hash_values_size < hash_size) {- free(galloc->hash_values);- galloc->hash_values = malloc(sizeof(struct hash_node) * hash_size);- galloc->hash_values_size = hash_size;+ // set the node_allocs from the hash table+ if (galloc->n_nodes < graph->n_nodes) {+ free(galloc->node_allocs);+ galloc->node_allocs = calloc(sizeof(struct node_alloc), graph->n_nodes);+ GGML_ASSERT(galloc->node_allocs != NULL);}-- // free hash_set.keys if needed- if (galloc->hash_set.keys != NULL) {- free(galloc->hash_set.keys);+ galloc->n_nodes = graph->n_nodes;+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ struct node_alloc * node_alloc = &galloc->node_allocs[i];+ node_alloc->buffer_id = get_node_buffer_id(node_buffer_ids, i);+ if (node->view_src || node->data) {+ node_alloc->dst.offset = SIZE_MAX;+ node_alloc->dst.size_max = 0;+ } else {+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);+ node_alloc->dst.offset = hn->offset;+ node_alloc->dst.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);+ }+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (!src || src->view_src || src->data) {+ node_alloc->src[j].offset = SIZE_MAX;+ node_alloc->src[j].size_max = 0;+ } else {+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, src);+ node_alloc->src[j].offset = hn->offset;+ node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src);+ }+ }+ }+ if (galloc->n_leafs < graph->n_leafs) {+ free(galloc->leaf_allocs);+ galloc->leaf_allocs = calloc(sizeof(struct tensor_alloc), graph->n_leafs);+ GGML_ASSERT(galloc->leaf_allocs != NULL);+ }+ galloc->n_leafs = graph->n_leafs;+ for (int i = 0; i < graph->n_leafs; i++) {+ struct ggml_tensor * leaf = graph->leafs[i];+ struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);+ galloc->leaf_allocs[i].offset = hn->offset;+ galloc->leaf_allocs[i].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);}- galloc->hash_set = hash_set;-- // reset hash values- memset(galloc->hash_values, 0, sizeof(struct hash_node) * hash_size);- galloc->hash_allocs = hash_node_talloc;+ // reallocate buffers if needed+ for (int i = 0; i < galloc->n_buffers; i++) {+ size_t cur_size = galloc->buffers[i] ? ggml_backend_buffer_get_size(galloc->buffers[i]) : 0;+ size_t new_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i]);- ggml_tallocr_alloc_graph_impl(galloc, graph);+ if (new_size > cur_size) {+#ifndef NDEBUG+ fprintf(stderr, "%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);+#endif+ ggml_backend_buffer_free(galloc->buffers[i]);+ galloc->buffers[i] = ggml_backend_buft_alloc_buffer(galloc->bufts[i], new_size);+ if (galloc->buffers[i] == NULL) {+ fprintf(stderr, "%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size);+ return false;+ }+ }+ }- // remove unowned resources- galloc->hash_set.keys = NULL;- galloc->hash_allocs = NULL;+ return true;}-// legacy API wrapper--struct ggml_allocr {- ggml_tallocr_t talloc;- ggml_gallocr_t galloc;-};--static ggml_allocr_t ggml_allocr_new_impl(ggml_tallocr_t talloc) {- ggml_allocr_t alloc = (ggml_allocr_t)malloc(sizeof(struct ggml_allocr));- *alloc = (struct ggml_allocr) {- /*.talloc = */ talloc,- /*.galloc = */ ggml_gallocr_new(),- };- return alloc;+bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) {+ return ggml_gallocr_reserve_n(galloc, graph, NULL);}-ggml_allocr_t ggml_allocr_new(void * data, size_t size, size_t alignment) {- return ggml_allocr_new_impl(ggml_tallocr_new(data, size, alignment));-}+static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, struct tensor_alloc * tensor_alloc) {+ assert(node->data || node->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max);-ggml_allocr_t ggml_allocr_new_measure(size_t alignment) {- return ggml_allocr_new_impl(ggml_tallocr_new_measure(alignment));+ if (node->view_src != NULL) {+ if (node->buffer == NULL) {+ assert(tensor_alloc->offset == SIZE_MAX);+ if (node->view_src->buffer == NULL) {+ // this tensor was allocated without ggml-backend+ return;+ }+ ggml_backend_view_init(galloc->buffers[buffer_id], node);+ }+ } else {+ if (node->data == NULL) {+ assert(tensor_alloc->offset != SIZE_MAX);+ assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max);+ void * base = ggml_backend_buffer_get_base(galloc->buffers[buffer_id]);+ void * addr = (char *)base + tensor_alloc->offset;+ ggml_backend_tensor_alloc(galloc->buffers[buffer_id], node, addr);+ } else {+ if (node->buffer == NULL) {+ // this tensor was allocated without ggml-backend+ return;+ }+ }+ }}-ggml_allocr_t ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer) {- return ggml_allocr_new_impl(ggml_tallocr_new_from_buffer(buffer));+static bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct node_alloc * nalloc, struct tensor_alloc * talloc) {+ ggml_backend_buffer_type_t buft = galloc->bufts[nalloc->buffer_id];+ size_t node_size = (node->data || node->view_src) ? 0 : ggml_backend_buft_get_alloc_size(buft, node);+ return talloc->size_max >= node_size;}-ggml_allocr_t ggml_allocr_new_from_backend(struct ggml_backend * backend, size_t size) {- return ggml_allocr_new_impl(ggml_tallocr_new_from_backend(backend, size));-}+static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {+ if (galloc->n_nodes != graph->n_nodes) {+#ifndef NDEBUG+ fprintf(stderr, "%s: graph has different number of nodes\n", __func__);+#endif+ return true;+ }-ggml_allocr_t ggml_allocr_new_measure_from_backend(struct ggml_backend * backend) {- return ggml_allocr_new_impl(ggml_tallocr_new_measure_from_backend(backend));-}+ if (galloc->n_leafs != graph->n_leafs) {+#ifndef NDEBUG+ fprintf(stderr, "%s: graph has different number of leafs\n", __func__);+#endif+ return true;+ }-struct ggml_backend_buffer * ggml_allocr_get_buffer(ggml_allocr_t alloc) {- return ggml_tallocr_get_buffer(alloc->talloc);-}+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ struct node_alloc * node_alloc = &galloc->node_allocs[i];-void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n) {- ggml_gallocr_set_parse_seq(alloc->galloc, list, n);-}+ if (!ggml_gallocr_node_needs_realloc(galloc, node, node_alloc, &node_alloc->dst)) {+#ifndef NDEBUG+ fprintf(stderr, "%s: node %s is not valid\n", __func__, node->name);+#endif+ return true;+ }-void ggml_allocr_free(ggml_allocr_t alloc) {- if (alloc == NULL) {- return;+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ continue;+ }+ if (!ggml_gallocr_node_needs_realloc(galloc, src, node_alloc, &node_alloc->src[j])) {+#ifndef NDEBUG+ fprintf(stderr, "%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name);+#endif+ return true;+ }+ }}- ggml_gallocr_free(alloc->galloc);- ggml_tallocr_free(alloc->talloc);- free(alloc);+ return false;}-bool ggml_allocr_is_measure(ggml_allocr_t alloc) {- return ggml_tallocr_is_measure(alloc->talloc);-}--void ggml_allocr_reset(ggml_allocr_t alloc) {- ggml_tallocr_reset(alloc->talloc);-}+bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {+ if (ggml_gallocr_needs_realloc(galloc, graph)) {+ if (galloc->n_buffers == 1) {+#ifndef NDEBUG+ fprintf(stderr, "%s: reallocating buffers automatically\n", __func__);+#endif+ if (!ggml_gallocr_reserve(galloc, graph)) {+ return false;+ }+ } else {+#ifndef NDEBUG+ fprintf(stderr, "%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__);+#endif+ return false;+ }+ }-void ggml_allocr_alloc(ggml_allocr_t alloc, struct ggml_tensor * tensor) {- ggml_tallocr_alloc(alloc->talloc, tensor);-}+ // reset buffers+ for (int i = 0; i < galloc->n_buffers; i++) {+ // zero size buffers are not allocated+ if (galloc->buffers[i] != NULL) {+ ggml_backend_buffer_reset(galloc->buffers[i]);+ }+ }-size_t ggml_allocr_max_size(ggml_allocr_t alloc) {- return ggml_tallocr_max_size(alloc->talloc);-}+ // allocate the graph tensors from the previous assignments+ // nodes+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ struct node_alloc * node_alloc = &galloc->node_allocs[i];+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ continue;+ }+ ggml_gallocr_init_tensor(galloc, src, node_alloc->buffer_id, &node_alloc->src[j]);+ }+ ggml_gallocr_init_tensor(galloc, node, node_alloc->buffer_id, &node_alloc->dst);+ }+ // leafs+ for (int i = 0; i < graph->n_leafs; i++) {+ struct ggml_tensor * leaf = graph->leafs[i];+ struct tensor_alloc * leaf_alloc = &galloc->leaf_allocs[i];+ ggml_gallocr_init_tensor(galloc, leaf, 0, leaf_alloc);+ }-size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph * graph) {- return ggml_gallocr_alloc_graph(alloc->galloc, alloc->talloc, graph);+ return true;}-// utils-ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {- GGML_ASSERT(ggml_get_no_alloc(ctx) == true);+size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) {+ GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers);- size_t alignment = ggml_backend_buft_get_alignment(buft);-- size_t nbytes = 0;- for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {- if (t->data == NULL && t->view_src == NULL) {- nbytes += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);- }+ if (galloc->buffers[buffer_id] == NULL) {+ return 0;}+ return ggml_backend_buffer_get_size(galloc->buffers[buffer_id]);+}- if (nbytes == 0) {- // all the tensors in the context are already allocated-#ifndef NDEBUG- fprintf(stderr, "%s: all tensors in the context are already allocated\n", __func__);-#endif- return NULL;- }+// utils- ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, nbytes);+static bool alloc_tensor_range(struct ggml_context * ctx,+ struct ggml_tensor * first, struct ggml_tensor * last,+ ggml_backend_buffer_type_t buft, size_t size,+ ggml_backend_buffer_t ** buffers, size_t * n_buffers) {+ ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);if (buffer == NULL) {- // failed to allocate buffer#ifndef NDEBUG- fprintf(stderr, "%s: failed to allocate buffer\n", __func__);+ fprintf(stderr, "%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), size);#endif- return NULL;+ for (size_t i = 0; i < *n_buffers; i++) {+ ggml_backend_buffer_free(*buffers[i]);+ }+ free(*buffers);+ return false;}- ggml_tallocr_t tallocr = ggml_tallocr_new_from_buffer(buffer);+ struct ggml_tallocr * tallocr = ggml_tallocr_new(buffer);- for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {+ for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {if (t->data == NULL) {if (t->view_src == NULL) {ggml_tallocr_alloc(tallocr, t);- } else {+ } else if (t->buffer == NULL) {ggml_backend_view_init(buffer, t);}} else {- if (t->view_src != NULL) {+ if (t->view_src != NULL && t->buffer == NULL) {// view of a pre-allocated tensorggml_backend_view_init(buffer, t);}@@ -828,6 +925,74 @@ ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_conteggml_tallocr_free(tallocr);+ *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1));+ (*buffers)[(*n_buffers)++] = buffer;++ return true;+}++ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {+ GGML_ASSERT(ggml_get_no_alloc(ctx) == true);++ size_t alignment = ggml_backend_buft_get_alignment(buft);+ size_t max_size = ggml_backend_buft_get_max_size(buft);++ ggml_backend_buffer_t * buffers = NULL;+ size_t n_buffers = 0;++ size_t cur_buf_size = 0;+ struct ggml_tensor * first = ggml_get_first_tensor(ctx);+ for (struct ggml_tensor * t = first; t != NULL; t = ggml_get_next_tensor(ctx, t)) {+ size_t this_size = 0;+ if (t->data == NULL && t->view_src == NULL) {+ this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);+ }++ if (this_size > max_size) {+ fprintf(stderr, "%s: tensor %s is too large to fit in a %s buffer (tensor size: %zu, max buffer size: %zu)\n",+ __func__, t->name,+ ggml_backend_buft_name(buft),+ this_size, max_size);+ for (size_t i = 0; i < n_buffers; i++) {+ ggml_backend_buffer_free(buffers[i]);+ }+ free(buffers);+ return NULL;+ }++ if ((cur_buf_size + this_size) > max_size) {+ // allocate tensors in the current buffer+ if (!alloc_tensor_range(ctx, first, t, buft, cur_buf_size, &buffers, &n_buffers)) {+ return NULL;+ }+ first = t;+ cur_buf_size = this_size;+ } else {+ cur_buf_size += this_size;+ }+ }++ // allocate remaining tensors+ if (cur_buf_size > 0) {+ if (!alloc_tensor_range(ctx, first, NULL, buft, cur_buf_size, &buffers, &n_buffers)) {+ return NULL;+ }+ }++ if (n_buffers == 0) {+#ifndef NDEBUG+ fprintf(stderr, "%s: all tensors in the context are already allocated\n", __func__);+#endif+ return NULL;+ }++ ggml_backend_buffer_t buffer;+ if (n_buffers == 1) {+ buffer = buffers[0];+ } else {+ buffer = ggml_backend_multi_buffer_alloc_buffer(buffers, n_buffers);+ }+ free(buffers);return buffer;}diff --git a/llama.cpp/ggml-alloc.h b/llama.cpp/ggml-alloc.hindex 4e599752..1d9085d1 100644--- a/llama.cpp/ggml-alloc.h+++ b/llama.cpp/ggml-alloc.h@@ -6,88 +6,62 @@extern "C" {#endif-struct ggml_backend;-struct ggml_backend_buffer;-struct ggml_backend_buffer_type;+typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;+typedef struct ggml_backend_buffer * ggml_backend_buffer_t;+typedef struct ggml_backend * ggml_backend_t;-//-// Legacy API-//--typedef struct ggml_allocr * ggml_allocr_t;--// initialize allocator for use with CPU backend only-GGML_API ggml_allocr_t ggml_allocr_new(void * data, size_t size, size_t alignment);-GGML_API ggml_allocr_t ggml_allocr_new_measure(size_t alignment);--// initialize allocator for use with ggml-backend-GGML_API ggml_allocr_t ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer);-GGML_API ggml_allocr_t ggml_allocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer-GGML_API ggml_allocr_t ggml_allocr_new_measure_from_backend(struct ggml_backend * backend);--GGML_API struct ggml_backend_buffer * ggml_allocr_get_buffer(ggml_allocr_t alloc);--// tell the allocator to parse nodes following the order described in the list-// you should call this if your graph are optimized to execute out-of-order-GGML_API void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n);--GGML_API void ggml_allocr_free (ggml_allocr_t alloc);-GGML_API bool ggml_allocr_is_measure (ggml_allocr_t alloc);-GGML_API void ggml_allocr_reset (ggml_allocr_t alloc);-GGML_API void ggml_allocr_alloc (ggml_allocr_t alloc, struct ggml_tensor * tensor);-GGML_API size_t ggml_allocr_max_size (ggml_allocr_t alloc);--GGML_API size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph * graph);+// Tensor allocator+typedef struct ggml_tallocr * ggml_tallocr_t;-//-// ggml-backend v2 API-//+GGML_API ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer);+GGML_API void ggml_tallocr_free(ggml_tallocr_t talloc);+GGML_API void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor);-// Separate tensor and graph allocator objects-// This is necessary for multi-backend allocation because the graph allocator needs to use multiple tensor allocators-// The original API is kept as a wrapper around the new API+// Graph allocator+/*+ Example usage:+ ggml_gallocr_t galloc = ggml_gallocr_new(ggml_bacckend_cpu_buffer_type());-// Tensor allocator-typedef struct ggml_tallocr * ggml_tallocr_t;+ // optional: create a worst-case graph and reserve the buffers to avoid reallocations+ ggml_gallocr_reserve(galloc, build_graph(max_batch));-GGML_API ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment);-GGML_API ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment);-GGML_API ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size);-GGML_API ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer-GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer);-GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft);-GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend);+ // allocate the graph+ struct ggml_cgraph * graph = build_graph(batch);+ ggml_gallocr_alloc_graph(galloc, graph);-GGML_API struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t talloc);+ printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0));-GGML_API void ggml_tallocr_free (ggml_tallocr_t talloc);-GGML_API bool ggml_tallocr_is_measure (ggml_tallocr_t talloc);-GGML_API void ggml_tallocr_reset (ggml_tallocr_t talloc);-GGML_API void ggml_tallocr_alloc (ggml_tallocr_t talloc, struct ggml_tensor * tensor);-GGML_API size_t ggml_tallocr_max_size (ggml_tallocr_t talloc);+ // evaluate the graph+ ggml_backend_graph_compute(backend, graph);+*/+// special tensor flags for use with the graph allocator:+// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses+// ggml_set_output(): output tensors are never freed and never overwritten-// Graph allocatortypedef struct ggml_gallocr * ggml_gallocr_t;-GGML_API ggml_gallocr_t ggml_gallocr_new(void);-GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);+GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);+GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);+GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);-GGML_API void ggml_gallocr_set_parse_seq(ggml_gallocr_t galloc, const int * list, int n);-GGML_API size_t ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, ggml_tallocr_t talloc, struct ggml_cgraph * graph);+// pre-allocate buffers from a measure graph - does not allocate or modify the graph+// call with a worst-case graph to avoid buffer reallocations+// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed+// returns false if the buffer allocation failed+GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);+GGML_API bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids);-// Allocate tensors from the allocators given by the hash table-GGML_API void ggml_gallocr_alloc_graph_n(- ggml_gallocr_t galloc,- struct ggml_cgraph * graph,- struct ggml_hash_set hash_set,- ggml_tallocr_t * hash_node_talloc);+// automatic reallocation if the topology changes when using a single buffer+// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)+GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);+GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);// Utils// Create a buffer and allocate all the tensors in a ggml_context-GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, struct ggml_backend_buffer_type * buft);-GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, struct ggml_backend * backend);+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);#ifdef __cplusplus}diff --git a/llama.cpp/ggml-backend-impl.h b/llama.cpp/ggml-backend-impl.hindex 3329168f..0e5b6850 100644--- a/llama.cpp/ggml-backend-impl.h+++ b/llama.cpp/ggml-backend-impl.h@@ -21,6 +21,7 @@ extern "C" {const char * (*GGML_CALL get_name) (ggml_backend_buffer_type_t buft);ggml_backend_buffer_t (*GGML_CALL alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);size_t (*GGML_CALL get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment+ size_t (*GGML_CALL get_max_size) (ggml_backend_buffer_type_t buft); // allocation max sizesize_t (*GGML_CALL get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including paddingbool (*GGML_CALL supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend// check if tensor data is in host memory@@ -65,6 +66,11 @@ extern "C" {// do not use directly, use ggml_backend_tensor_copy insteadbool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);+ // buffer that contains a collection of buffers+ GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);+ GGML_CALL bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);+ GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);+//// Backend//@@ -127,7 +133,6 @@ extern "C" {void *(*GGML_CALL malloc)(size_t);void (*GGML_CALL ggml_backend_register)(const char *, ggml_backend_init_fn, ggml_backend_buffer_type_t, void *);ggml_backend_buffer_t (*GGML_CALL ggml_backend_buffer_init)(ggml_backend_buffer_type_t, struct ggml_backend_buffer_i, ggml_backend_buffer_context_t, size_t);- bool (*GGML_CALL ggml_backend_buffer_is_host)(ggml_backend_buffer_t);ggml_backend_buffer_t (*GGML_CALL ggml_backend_cpu_buffer_from_ptr)(void *, size_t);ggml_backend_buffer_type_t (*GGML_CALL ggml_backend_cpu_buffer_type)(void);size_t (*GGML_CALL ggml_backend_buft_get_alloc_size)(ggml_backend_buffer_type_t, struct ggml_tensor *);@@ -151,6 +156,7 @@ extern "C" {size_t (*GGML_CALL ggml_row_size)(enum ggml_type, int64_t);void (*GGML_CALL ggml_rope_yarn_corr_dims)(int, int, float, float, float, float[2]);const char *(*GGML_CALL ggml_op_desc)(const struct ggml_tensor *);+ bool (*GGML_CALL ggml_backend_buffer_is_host)(ggml_backend_buffer_t);};#ifdef __cplusplusdiff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex 0c9d0c03..e21683ad 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -17,6 +17,10 @@#include "ggml-metal.h"#include "llamafile/log.h"+#ifndef NDEBUG+#define NDEBUG // [jart] delete printf debugging+#endif+#define MAX(a, b) ((a) > (b) ? (a) : (b))@@ -35,6 +39,14 @@ size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {return buft->iface.get_alignment(buft);}+size_t ggml_backend_buft_get_max_size(ggml_backend_buffer_type_t buft) {+ // get_max_size is optional, defaults to SIZE_MAX+ if (buft->iface.get_max_size) {+ return buft->iface.get_max_size(buft);+ }+ return SIZE_MAX;+}+GGML_CALL size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) {// get_alloc_size is optional, defaults to ggml_nbytesif (buft->iface.get_alloc_size) {@@ -65,8 +77,6 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(size_t size) {ggml_backend_buffer_t buffer = malloc(sizeof(struct ggml_backend_buffer));- GGML_ASSERT(iface.get_base != NULL);-(*buffer) = (struct ggml_backend_buffer) {/* .interface = */ iface,/* .buft = */ buft,@@ -116,6 +126,10 @@ size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer) {return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));}+size_t ggml_backend_buffer_get_max_size(ggml_backend_buffer_t buffer) {+ return ggml_backend_buft_get_max_size(ggml_backend_buffer_get_type(buffer));+}+size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);}@@ -130,6 +144,11 @@ GGML_CALL bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {buffer->usage = usage;++ // FIXME: add a generic callback to the buffer interface+ if (ggml_backend_buffer_is_multi_buffer(buffer)) {+ ggml_backend_multi_buffer_set_usage(buffer, usage);+ }}ggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {@@ -179,6 +198,10 @@ size_t ggml_backend_get_alignment(ggml_backend_t backend) {return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));}+size_t ggml_backend_get_max_size(ggml_backend_t backend) {+ return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));+}+void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");@@ -208,6 +231,10 @@ GGML_CALL void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void *GGML_ASSERT(buf != NULL && "tensor buffer not set");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");+ if (!size) {+ return;+ }+tensor->buffer->iface.set_tensor(buf, tensor, data, offset, size);}@@ -218,6 +245,10 @@ GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void *GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");+ if (!size) {+ return;+ }+tensor->buffer->iface.get_tensor(buf, tensor, data, offset, size);}@@ -342,12 +373,31 @@ GGML_CALL static void ggml_backend_registry_init(void) {ggml_backend_register("CPU", ggml_backend_reg_cpu_init, ggml_backend_cpu_buffer_type(), NULL);// add forward decls here to avoid including the backend headers- extern GGML_CALL void ggml_backend_cuda_reg_devices(void);+/* #ifdef GGML_USE_CUBLAS */+ extern GGML_API GGML_CALL void ggml_backend_cuda_reg_devices(void);ggml_backend_cuda_reg_devices();+/* #endif */++#ifdef GGML_USE_SYCL+ extern void ggml_backend_sycl_reg_devices(void);+ ggml_backend_sycl_reg_devices();+#endif+/* #ifdef GGML_USE_METAL */extern GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data);extern GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);ggml_backend_register("Metal", ggml_backend_reg_metal_init, ggml_backend_metal_buffer_type(), NULL);+/* #endif */++#ifdef GGML_USE_VULKAN+ extern GGML_CALL int ggml_backend_vk_reg_devices(void);+ ggml_backend_vk_reg_devices();+#endif++#ifdef GGML_USE_KOMPUTE+ extern GGML_CALL void ggml_backend_kompute_reg_devices(void);+ ggml_backend_kompute_reg_devices();+#endif}GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {@@ -445,6 +495,8 @@ ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size) {// backend CPU+static const size_t TENSOR_ALIGNMENT = 32; // required for mmap as gguf only guarantees 32-byte alignment+GGML_CALL static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t buffer) {return "CPU";@@ -452,7 +504,14 @@ GGML_CALL static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t}GGML_CALL static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {- return (void *)buffer->context;+ uintptr_t data = (uintptr_t)buffer->context;++ // align the buffer+ if (data % TENSOR_ALIGNMENT != 0) {+ data = GGML_PAD(data, TENSOR_ALIGNMENT);+ }++ return (void *)data;}GGML_CALL static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {@@ -510,8 +569,6 @@ static struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = {/* .reset = */ NULL,};-static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512-GGML_CALL static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {return "CPU";@@ -520,9 +577,11 @@ GGML_CALL static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backendGGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {size += TENSOR_ALIGNMENT; // malloc may return an address that is not aligned- void * data = malloc(size); // TODO: maybe use GGML_ALIGNED_MALLOC?-- GGML_ASSERT(data != NULL && "failed to allocate buffer");+ void * data = malloc(size); // TODO: use GGML_ALIGNED_MALLOC (move to ggml-impl.h)+ if (data == NULL) {+ fprintf(stderr, "%s: failed to allocate buffer of size %zu\n", __func__, size);+ return NULL;+ }return ggml_backend_buffer_init(buft, cpu_backend_buffer_i, data, size);}@@ -551,6 +610,7 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {/* .get_name = */ ggml_backend_cpu_buffer_type_get_name,/* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes/* .supports_backend = */ ggml_backend_cpu_buffer_type_supports_backend,/* .is_host = */ ggml_backend_cpu_buffer_type_is_host,@@ -606,6 +666,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {/* .get_name = */ ggml_backend_cpu_hbm_buffer_type_get_name,/* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes/* .supports_backend = */ ggml_backend_cpu_buffer_type_supports_backend,/* .is_host = */ ggml_backend_cpu_buffer_type_is_host,@@ -621,6 +682,9 @@ struct ggml_backend_cpu_context {int n_threads;void * work_data;size_t work_size;++ ggml_abort_callback abort_callback;+ void * abort_callback_data;};GGML_CALL static const char * ggml_backend_cpu_name(ggml_backend_t backend) {@@ -659,6 +723,9 @@ GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggcpu_plan->cplan.work_data = malloc(cpu_plan->cplan.work_size);}+ cpu_plan->cplan.abort_callback = cpu_ctx->abort_callback;+ cpu_plan->cplan.abort_callback_data = cpu_ctx->abort_callback_data;+return cpu_plan;}@@ -689,9 +756,11 @@ GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, strcpu_ctx->work_data = realloc(cpu_ctx->work_data, cplan.work_size);cpu_ctx->work_size = cplan.work_size;}-cplan.work_data = cpu_ctx->work_data;+ cplan.abort_callback = cpu_ctx->abort_callback;+ cplan.abort_callback_data = cpu_ctx->abort_callback_data;+ggml_graph_compute(cgraph, &cplan);return true;}@@ -699,7 +768,7 @@ GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, strGGML_CALL static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {switch (op->op) {case GGML_OP_CPY:- return op->type != GGML_TYPE_IQ2_XXS && op->type != GGML_TYPE_IQ2_XS; // missing type_traits.from_float+ return op->type != GGML_TYPE_IQ2_XXS && op->type != GGML_TYPE_IQ2_XS && op->type != GGML_TYPE_IQ1_S; // missing type_traits.from_floatcase GGML_OP_MUL_MAT:return op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == ggml_internal_get_type_traits(op->src[0]->type).vec_dot_type;default:@@ -726,12 +795,21 @@ static struct ggml_backend_i cpu_backend_i = {ggml_backend_t ggml_backend_cpu_init(void) {struct ggml_backend_cpu_context * ctx = malloc(sizeof(struct ggml_backend_cpu_context));+ if (ctx == NULL) {+ return NULL;+ }- ctx->n_threads = GGML_DEFAULT_N_THREADS;- ctx->work_data = NULL;- ctx->work_size = 0;+ ctx->n_threads = GGML_DEFAULT_N_THREADS;+ ctx->work_data = NULL;+ ctx->work_size = 0;+ ctx->abort_callback = NULL;+ ctx->abort_callback_data = NULL;ggml_backend_t cpu_backend = malloc(sizeof(struct ggml_backend));+ if (cpu_backend == NULL) {+ free(ctx);+ return NULL;+ }*cpu_backend = (struct ggml_backend) {/* .interface = */ cpu_backend_i,@@ -751,7 +829,16 @@ void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {ctx->n_threads = n_threads;}+void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data) {+ GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));++ struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;+ ctx->abort_callback = abort_callback;+ ctx->abort_callback_data = abort_callback_data;+}+GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {+ GGML_ASSERT((uintptr_t)ptr % TENSOR_ALIGNMENT == 0 && "buffer pointer must be aligned");return ggml_backend_buffer_init(ggml_backend_cpu_buffer_type(), cpu_backend_buffer_i_from_ptr, ptr, size);}@@ -762,6 +849,94 @@ GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, vGGML_UNUSED(user_data);}+// multi-buffer buffer++struct ggml_backend_multi_buffer_context {+ ggml_backend_buffer_t * buffers;+ size_t n_buffers;+};++typedef struct ggml_backend_multi_buffer_context * ggml_backend_multi_buffer_context_t;++GGML_CALL static const char * ggml_backend_multi_buffer_get_name(ggml_backend_buffer_t buffer) {+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;++ return ctx->buffers[0]->iface.get_name(ctx->buffers[0]);+}++GGML_CALL static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;+ for (size_t i = 0; i < ctx->n_buffers; i++) {+ ggml_backend_buffer_free(ctx->buffers[i]);+ }++ free(ctx->buffers);+ free(ctx);+}++GGML_CALL static void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;+ for (size_t i = 0; i < ctx->n_buffers; i++) {+ ggml_backend_buffer_clear(ctx->buffers[i], value);+ }+}++static struct ggml_backend_buffer_i ggml_backend_multi_buffer_context_interface(void) {+ static struct ggml_backend_buffer_i multi_backend_buffer_i = {+ /* .get_name = */ ggml_backend_multi_buffer_get_name,+ /* .free_buffer = */ ggml_backend_multi_buffer_free_buffer,+ /* .get_base = */ NULL,+ /* .init_tensor = */ NULL,+ /* .set_tensor = */ NULL,+ /* .get_tensor = */ NULL,+ /* .cpy_tensor = */ NULL,+ /* .clear = */ ggml_backend_multi_buffer_clear,+ /* .reset = */ NULL,+ };++ return multi_backend_buffer_i;+}++GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) malloc(sizeof(struct ggml_backend_multi_buffer_context));+ ctx->n_buffers = n_buffers;+ ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t));++ GGML_ASSERT(ctx->buffers != NULL);++ size_t total_size = 0;+ for (size_t i = 0; i < n_buffers; i++) {+ ctx->buffers[i] = buffers[i];+ total_size += ggml_backend_buffer_get_size(buffers[i]);+ }++ return ggml_backend_buffer_init(buffers[0]->buft, ggml_backend_multi_buffer_context_interface(), ctx, total_size);+}++GGML_CALL bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer) {+ return buffer->iface.get_name == ggml_backend_multi_buffer_get_name;+}++GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {+ GGML_ASSERT(ggml_backend_buffer_is_multi_buffer(buffer));+ ggml_backend_multi_buffer_context_t ctx = (ggml_backend_multi_buffer_context_t) buffer->context;+ for (size_t i = 0; i < ctx->n_buffers; i++) {+ ggml_backend_buffer_set_usage(ctx->buffers[i], usage);+ }+}++// creates a copy of the tensor with the same memory layout+static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {+ struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);+ for (int i = 0; i < GGML_MAX_DIMS; i++) {+ dup->nb[i] = tensor->nb[i];+ }+ return dup;+}++static bool ggml_is_view_op(enum ggml_op op) {+ return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;+}// scheduler@@ -770,7 +945,7 @@ GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, v#define GGML_MAX_SPLIT_INPUTS 16struct ggml_backend_sched_split {- ggml_tallocr_t tallocr;+ int backend_id;int i_start;int i_end;struct ggml_tensor * inputs[GGML_MAX_SPLIT_INPUTS];@@ -785,15 +960,17 @@ struct ggml_backend_sched {int n_backends;ggml_backend_t backends[GGML_MAX_BACKENDS];ggml_backend_buffer_type_t bufts[GGML_MAX_BACKENDS];- ggml_tallocr_t tallocs[GGML_MAX_BACKENDS];ggml_gallocr_t galloc;// hash keys of the nodes in the graphstruct ggml_hash_set hash_set;- // hash values (arrays of [hash_set.size])- ggml_tallocr_t * node_talloc; // tallocr assigned to each node (indirectly this is the backend)- struct ggml_tensor * (* node_copies)[GGML_MAX_BACKENDS]; // copies of each node for each destination backend+ // hash values+ int * tensor_backend_id;+ struct ggml_tensor * (* tensor_copies)[GGML_MAX_BACKENDS];++ int * node_backend_ids; // [n_nodes]+ int n_nodes;// copy of the graph with modified inputsstruct ggml_cgraph * graph;@@ -803,75 +980,45 @@ struct ggml_backend_sched {struct ggml_context * ctx;+ ggml_backend_sched_eval_callback callback_eval;+ void * callback_eval_user_data;+// align context_buffer to GGML_MEM_ALIGN#ifdef _MSC_VER__declspec(align(GGML_MEM_ALIGN))#else__attribute__((aligned(GGML_MEM_ALIGN)))#endif- char context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)];-- ggml_backend_sched_eval_callback callback_eval;- void * callback_eval_user_data;+ char context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)];};#define hash_id(node) ggml_hash_find_or_insert(sched->hash_set, node)-#define node_allocr(node) sched->node_talloc[hash_id(node)]--static bool ggml_is_view_op(enum ggml_op op) {- return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;-}+#define tensor_backend_id(node) sched->tensor_backend_id[hash_id(node)]+#define tensor_backend(node) (tensor_backend_id(node) == -1 ? NULL : sched->backends[tensor_backend_id(node)])-// returns the priority of the backend, lower is better-static int sched_backend_prio(ggml_backend_sched_t sched, ggml_backend_t backend) {+// returns the priority of the backend, lower id is higher priority+static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {for (int i = 0; i < sched->n_backends; i++) {if (sched->backends[i] == backend) {return i;}}- return INT_MAX;-}--static int sched_allocr_prio(ggml_backend_sched_t sched, ggml_tallocr_t allocr) {- for (int i = 0; i < sched->n_backends; i++) {- if (sched->tallocs[i] == allocr) {- return i;- }- }- return INT_MAX;+ return -1;}-static ggml_tallocr_t sched_allocr_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) {+static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) {if (buffer == NULL) {- return NULL;- }-- // check if this is already allocate in a allocr buffer (from user manual allocations)- for (int i = 0; i < sched->n_backends; i++) {- if (ggml_tallocr_get_buffer(sched->tallocs[i]) == buffer) {- return sched->tallocs[i];- }+ return -1;}// find highest prio backend that supports the buffer typefor (int i = 0; i < sched->n_backends; i++) {if (ggml_backend_buft_supports_backend(buffer->buft, sched->backends[i])) {- return sched->tallocs[i];+ return i;}}GGML_ASSERT(false && "tensor buffer type not supported by any backend");-}--static ggml_backend_t get_allocr_backend(ggml_backend_sched_t sched, ggml_tallocr_t allocr) {- if (allocr == NULL) {- return NULL;- }- for (int i = 0; i < sched->n_backends; i++) {- if (sched->tallocs[i] == allocr) {- return sched->backends[i];- }- }- GGML_UNREACHABLE();+ return -1; // silence warning}#if 0@@ -884,37 +1031,39 @@ static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_I#endif// returns the backend that should be used for the node based on the current locations-static ggml_tallocr_t sched_allocr_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * node) {+static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) {+ // TODO: use supports_op to check if the backend supports the op+// assign pre-allocated nodes to their backend// dst- ggml_tallocr_t cur_allocr = sched_allocr_from_buffer(sched, node->buffer);- if (cur_allocr != NULL) {+ int cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->buffer);+ if (cur_backend != -1) {SET_CAUSE(node, "1.dst");- return cur_allocr;+ return cur_backend;}// view_src- if (node->view_src != NULL) {- cur_allocr = sched_allocr_from_buffer(sched, node->view_src->buffer);- if (cur_allocr != NULL) {+ if (tensor->view_src != NULL) {+ cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src->buffer);+ if (cur_backend != -1) {SET_CAUSE(node, "1.vsrc");- return cur_allocr;+ return cur_backend;}}// assign nodes that use weights to the backend of the weightsfor (int i = 0; i < GGML_MAX_SRC; i++) {- const struct ggml_tensor * src = node->src[i];+ const struct ggml_tensor * src = tensor->src[i];if (src == NULL) {- break;+ continue;}if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {- ggml_tallocr_t src_allocr = sched_allocr_from_buffer(sched, src->buffer);+ int src_backend = ggml_backend_sched_backend_from_buffer(sched, src->buffer);// operations with weights are always run on the same backend as the weightsSET_CAUSE(node, "1.wgt%d", i);- return src_allocr;+ return src_backend;}}- return NULL;+ return -1;}static char * fmt_size(size_t size) {@@ -927,11 +1076,11 @@ static char * fmt_size(size_t size) {return buffer;}-static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {int cur_split = 0;for (int i = 0; i < graph->n_nodes; i++) {if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {- ggml_backend_t split_backend = get_allocr_backend(sched, sched->splits[cur_split].tallocr);+ ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];fprintf(stderr, "\n## SPLIT #%d: %s # %d inputs: ", cur_split, ggml_backend_name(split_backend),sched->splits[cur_split].n_inputs);for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {@@ -945,17 +1094,15 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraif (ggml_is_view_op(node->op)) {continue;}- ggml_tallocr_t node_allocr = node_allocr(node);- ggml_backend_t node_backend = node_allocr ? get_allocr_backend(sched, node_allocr) : NULL; // FIXME:+ ggml_backend_t tensor_backend = tensor_backend(node);fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, ggml_op_name(node->op), node->name,- fmt_size(ggml_nbytes(node)), node_allocr ? ggml_backend_name(node_backend) : "NULL", GET_CAUSE(node));+ fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node));for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {- break;+ continue;}- ggml_tallocr_t src_allocr = node_allocr(src);- ggml_backend_t src_backend = src_allocr ? get_allocr_backend(sched, src_allocr) : NULL;+ ggml_backend_t src_backend = tensor_backend(src);fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name,fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));}@@ -963,23 +1110,13 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgra}}-// creates a copy of the tensor with the same memory layout-static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {- struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);- for (int i = 0; i < GGML_MAX_DIMS; i++) {- dup->nb[i] = tensor->nb[i];- }- return dup;-}--//#define DEBUG_PASS1//#define DEBUG_PASS2//#define DEBUG_PASS3//#define DEBUG_PASS4// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend-static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {// reset splitssched->n_splits = 0;sched->is_reset = false;@@ -1001,28 +1138,28 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// pass 1: assign backends to ops with pre-allocated inputsfor (int i = 0; i < graph->n_leafs; i++) {struct ggml_tensor * leaf = graph->leafs[i];- if (node_allocr(leaf) != NULL) {+ if (tensor_backend_id(leaf) != -1) {// do not overwrite user assignmentscontinue;}- node_allocr(leaf) = sched_allocr_from_cur(sched, leaf);+ tensor_backend_id(leaf) = ggml_backend_sched_backend_id_from_cur(sched, leaf);}for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- if (node_allocr(node) != NULL) {+ if (tensor_backend_id(node) != -1) {// do not overwrite user assignmentscontinue;}- node_allocr(node) = sched_allocr_from_cur(sched, node);+ tensor_backend_id(node) = ggml_backend_sched_backend_id_from_cur(sched, node);// srcfor (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {- break;+ continue;}- if (node_allocr(src) == NULL) {- node_allocr(src) = sched_allocr_from_cur(sched, src);+ if (tensor_backend_id(src) == -1) {+ tensor_backend_id(src) = ggml_backend_sched_backend_id_from_cur(sched, src);}}}@@ -1037,22 +1174,22 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// pass 2.1 expand gpu up{- ggml_tallocr_t cur_allocr = NULL;+ int cur_backend_id = -1;for (int i = graph->n_nodes - 1; i >= 0; i--) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- ggml_tallocr_t node_allocr = node_allocr(node);- if (node_allocr != NULL) {- if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) {+ int tensor_backend_id = tensor_backend_id(node);+ if (tensor_backend_id != -1) {+ if (tensor_backend_id == sched->n_backends - 1) {// skip cpu (lowest prio backend)- cur_allocr = NULL;+ cur_backend_id = -1;} else {- cur_allocr = node_allocr;+ cur_backend_id = tensor_backend_id;}} else {- node_allocr(node) = cur_allocr;+ tensor_backend_id(node) = cur_backend_id;SET_CAUSE(node, "2.1");}}@@ -1060,22 +1197,22 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// pass 2.2 expand gpu down{- ggml_tallocr_t cur_allocr = NULL;+ int cur_backend_id = -1;for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- ggml_tallocr_t node_allocr = node_allocr(node);- if (node_allocr != NULL) {- if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) {+ int tensor_backend_id = tensor_backend_id(node);+ if (tensor_backend_id != -1) {+ if (tensor_backend_id == sched->n_backends - 1) {// skip cpu (lowest prio backend)- cur_allocr = NULL;+ cur_backend_id = -1;} else {- cur_allocr = node_allocr;+ cur_backend_id = tensor_backend_id;}} else {- node_allocr(node) = cur_allocr;+ tensor_backend_id(node) = cur_backend_id;SET_CAUSE(node, "2.2");}}@@ -1083,17 +1220,17 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// pass 2.3 expand rest up{- ggml_tallocr_t cur_allocr = NULL;+ int cur_backend_id = -1;for (int i = graph->n_nodes - 1; i >= 0; i--) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- ggml_tallocr_t node_allocr = node_allocr(node);- if (node_allocr != NULL) {- cur_allocr = node_allocr;+ int tensor_backend_id = tensor_backend_id(node);+ if (tensor_backend_id != -1) {+ cur_backend_id = tensor_backend_id;} else {- node_allocr(node) = cur_allocr;+ tensor_backend_id(node) = cur_backend_id;SET_CAUSE(node, "2.3");}}@@ -1101,17 +1238,17 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// pass 2.4 expand rest down{- ggml_tallocr_t cur_allocr = NULL;+ int cur_backend_id = -1;for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- ggml_tallocr_t node_allocr = node_allocr(node);- if (node_allocr != NULL) {- cur_allocr = node_allocr;+ int tensor_backend_id = tensor_backend_id(node);+ if (tensor_backend_id != -1) {+ cur_backend_id = tensor_backend_id;} else {- node_allocr(node) = cur_allocr;+ tensor_backend_id(node) = cur_backend_id;SET_CAUSE(node, "2.4");}}@@ -1123,24 +1260,24 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// pass 3: assign backends to remaining src from dst and view_srcfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- ggml_tallocr_t cur_allocr = node_allocr(node);- if (node->view_src != NULL && cur_allocr == NULL) {- cur_allocr = node_allocr(node) = node_allocr(node->view_src);+ int cur_backend_id = tensor_backend_id(node);+ if (node->view_src != NULL && cur_backend_id == -1) {+ cur_backend_id = tensor_backend_id(node) = tensor_backend_id(node->view_src);SET_CAUSE(node, "3.vsrc");}for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {- break;+ continue;}- ggml_tallocr_t src_allocr = node_allocr(src);- if (src_allocr == NULL) {+ int src_backend_id = tensor_backend_id(src);+ if (src_backend_id == -1) {if (src->view_src != NULL) {// views are always on the same backend as the source- node_allocr(src) = node_allocr(src->view_src);+ tensor_backend_id(src) = tensor_backend_id(src->view_src);SET_CAUSE(src, "3.vsrc");} else {- node_allocr(src) = cur_allocr;+ tensor_backend_id(src) = cur_backend_id;SET_CAUSE(src, "3.cur");}}@@ -1157,15 +1294,14 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * gfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];if (!ggml_is_view_op(node->op)) {- sched->splits[0].tallocr = node_allocr(node);+ sched->splits[0].backend_id = tensor_backend_id(node);break;}}sched->splits[0].i_start = 0;sched->splits[0].n_inputs = 0;memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK- ggml_tallocr_t cur_allocr = sched->splits[0].tallocr;- size_t cur_backend_id = sched_allocr_prio(sched, cur_allocr);+ int cur_backend_id = sched->splits[0].backend_id;for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];@@ -1173,64 +1309,45 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * gcontinue;}- ggml_tallocr_t node_allocr = node_allocr(node);+ int tensor_backend_id = tensor_backend_id(node);- GGML_ASSERT(node_allocr != NULL); // all nodes should be assigned by now+ GGML_ASSERT(tensor_backend_id != -1); // all nodes should be assigned by now- if (node_allocr != cur_allocr) {+ if (tensor_backend_id != cur_backend_id) {sched->splits[cur_split].i_end = i;cur_split++;GGML_ASSERT(cur_split < GGML_MAX_SPLITS);- sched->splits[cur_split].tallocr = node_allocr;+ sched->splits[cur_split].backend_id = tensor_backend_id;sched->splits[cur_split].i_start = i;sched->splits[cur_split].n_inputs = 0;- cur_allocr = node_allocr;- cur_backend_id = sched_allocr_prio(sched, cur_allocr);+ cur_backend_id = tensor_backend_id;}// find inputs that are not on the same backendfor (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {- break;+ continue;}- ggml_tallocr_t src_allocr = node_allocr(src);- GGML_ASSERT(src_allocr != NULL); // all inputs should be assigned by now- if (src_allocr != node_allocr) {+ int src_backend_id = tensor_backend_id(src);+ assert(src_backend_id != -1); // all inputs should be assigned by now+ if (src_backend_id != tensor_backend_id) {// create a copy of the input in the split's backendsize_t id = hash_id(src);- if (sched->node_copies[id][cur_backend_id] == NULL) {- ggml_backend_t backend = get_allocr_backend(sched, cur_allocr);+ if (sched->tensor_copies[id][cur_backend_id] == NULL) {+ ggml_backend_t backend = sched->backends[cur_backend_id];struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name);- sched->node_copies[id][cur_backend_id] = tensor_copy;- node_allocr(tensor_copy) = cur_allocr;+ sched->tensor_copies[id][cur_backend_id] = tensor_copy;+ tensor_backend_id(tensor_copy) = cur_backend_id;SET_CAUSE(tensor_copy, "4.cpy");int n_inputs = sched->splits[cur_split].n_inputs++;GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS);sched->splits[cur_split].inputs[n_inputs] = src;}- node->src[j] = sched->node_copies[id][cur_backend_id];--#if 0- // check if the input is already in the split- bool found = false;- for (int k = 0; k < sched->splits[cur_split].n_inputs; k++) {- if (sched->splits[cur_split].inputs[k] == src) {- found = true;- break;- }- }-- if (!found) {- int n_inputs = sched->splits[cur_split].n_inputs++;- //printf("split %d input %d: %s (%s)\n", cur_split, n_inputs, src->name, ggml_backend_name(get_allocr_backend(sched, src_allocr)));- GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS);- sched->splits[cur_split].inputs[n_inputs] = src;- }-#endif+ node->src[j] = sched->tensor_copies[id][cur_backend_id];}}}@@ -1245,30 +1362,30 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// sanity check: all sources should have the same backend as the nodefor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- ggml_tallocr_t node_allocr = node_allocr(node);- if (node_allocr == NULL) {+ ggml_backend_t tensor_backend = tensor_backend(node);+ if (tensor_backend == NULL) {fprintf(stderr, "!!!!!!! %s has no backend\n", node->name);}- if (node->view_src != NULL && node_allocr != node_allocr(node->view_src)) {+ if (node->view_src != NULL && tensor_backend != tensor_backend(node->view_src)) {fprintf(stderr, "!!!!!!! %s has backend %s, view_src %s has backend %s\n",- node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL",- node->view_src->name, node_allocr(node->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(node->view_src))) : "NULL");+ node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL",+ node->view_src->name, tensor_backend(node->view_src) ? ggml_backend_name(tensor_backend(node->view_src)) : "NULL");}for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {- break;+ continue;}- ggml_tallocr_t src_allocr = node_allocr(src);- if (src_allocr != node_allocr /* && src_backend != NULL */) { // ignore nulls for now+ ggml_backend_t src_backend = tensor_backend(src);+ if (src_backend != tensor_backend /* && src_backend != NULL */) {fprintf(stderr, "!!!! %s has backend %s, src %d (%s) has backend %s\n",- node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL",- j, src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL");+ node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL",+ j, src->name, src_backend ? ggml_backend_name(src_backend) : "NULL");}- if (src->view_src != NULL && src_allocr != node_allocr(src->view_src)) {+ if (src->view_src != NULL && src_backend != tensor_backend(src->view_src)) {fprintf(stderr, "!!!!!!! [src] %s has backend %s, view_src %s has backend %s\n",- src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL",- src->view_src->name, node_allocr(src->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(src->view_src))) : "NULL");+ src->name, src_backend ? ggml_backend_name(src_backend) : "NULL",+ src->view_src->name, tensor_backend(src->view_src) ? ggml_backend_name(tensor_backend(src->view_src)) : "NULL");}}}@@ -1282,32 +1399,45 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * gstruct ggml_backend_sched_split * split = &sched->splits[i];split->graph = ggml_graph_view(graph, split->i_start, split->i_end);- // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the splitfor (int j = 0; j < split->n_inputs; j++) {struct ggml_tensor * input = split->inputs[j];- struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_allocr_prio(sched, split->tallocr)];+ struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split->backend_id];+// add a dependency to the input source so that it is not freed before the copy is done- GGML_ASSERT(input_cpy->src[0] == NULL || input_cpy->src[0] == input);- input_cpy->src[0] = input;+ struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input);+ sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(input);+ graph_copy->nodes[graph_copy->n_nodes++] = input_dep;++ // add a dependency to the input copy so that it is allocated at the start of the split+ sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;graph_copy->nodes[graph_copy->n_nodes++] = input_cpy;}for (int j = split->i_start; j < split->i_end; j++) {+ sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];}}sched->graph = graph_copy;}-static void sched_alloc_splits(ggml_backend_sched_t sched) {- ggml_gallocr_alloc_graph_n(- sched->galloc,- sched->graph,- sched->hash_set,- sched->node_talloc);+static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {+ // ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids);+ if (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) {+#ifndef NDEBUG+ fprintf(stderr, "ggml_backend_sched: failed to allocate graph, reserving\n");+#endif+ ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids);+ if (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) {+ fprintf(stderr, "ggml_backend_sched: failed to allocate graph\n");+ return false;+ }+ }++ return true;}-static void sched_compute_splits(ggml_backend_sched_t sched) {+static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {uint64_t copy_us[GGML_MAX_BACKENDS] = {0};uint64_t compute_us[GGML_MAX_BACKENDS] = {0};@@ -1315,20 +1445,18 @@ static void sched_compute_splits(ggml_backend_sched_t sched) {for (int i = 0; i < sched->n_splits; i++) {struct ggml_backend_sched_split * split = &splits[i];- ggml_backend_t split_backend = get_allocr_backend(sched, split->tallocr);- int split_backend_id = sched_backend_prio(sched, split_backend);+ int split_backend_id = split->backend_id;+ ggml_backend_t split_backend = sched->backends[split_backend_id];// copy the input tensors to the split backenduint64_t copy_start_us = ggml_time_us();for (int j = 0; j < split->n_inputs; j++) {struct ggml_tensor * input = split->inputs[j];- struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][split_backend_id];+ struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split_backend_id];GGML_ASSERT(input->buffer != NULL);GGML_ASSERT(input_cpy->buffer != NULL);- // TODO: avoid this copy if it was already copied in a previous split, and the input didn't change- // this is important to avoid copying constants such as KQ_mask and inp_pos multiple timesggml_backend_tensor_copy_async(split_backend, input, input_cpy);}//ggml_backend_synchronize(split_backend); // necessary to measure copy time@@ -1344,7 +1472,9 @@ static void sched_compute_splits(ggml_backend_sched_t sched) {uint64_t compute_start_us = ggml_time_us();if (!sched->callback_eval) {- ggml_backend_graph_compute(split_backend, &split->graph);+ if (!ggml_backend_graph_compute(split_backend, &split->graph)) {+ return false;+ }//ggml_backend_synchronize(split_backend); // necessary to measure compute time} else {// similar to ggml_backend_compare_graph_backend@@ -1364,7 +1494,9 @@ static void sched_compute_splits(ggml_backend_sched_t sched) {struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);- ggml_backend_graph_compute(split_backend, &gv);+ if (!ggml_backend_graph_compute(split_backend, &gv)) {+ return false;+ }if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) {break;@@ -1386,19 +1518,8 @@ static void sched_compute_splits(ggml_backend_sched_t sched) {}}#endif-}-static void sched_reset(ggml_backend_sched_t sched) {- for (int i = 0; i < sched->n_backends; i++) {- ggml_tallocr_reset(sched->tallocs[i]);- }- // reset state for the next run- size_t hash_size = sched->hash_set.size;- memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size);- memset(sched->node_talloc, 0, sizeof(sched->node_talloc[0]) * hash_size);- memset(sched->node_copies, 0, sizeof(sched->node_copies[0]) * hash_size);-- sched->is_reset = true;+ return true;}ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size) {@@ -1408,9 +1529,10 @@ ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backstruct ggml_backend_sched * sched = calloc(sizeof(struct ggml_backend_sched), 1);// initialize hash table- sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);- sched->node_talloc = calloc(sizeof(sched->node_talloc[0]) * sched->hash_set.size, 1);- sched->node_copies = calloc(sizeof(sched->node_copies[0]) * sched->hash_set.size, 1);+ sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);+ sched->tensor_backend_id = calloc(sizeof(sched->tensor_backend_id[0]), sched->hash_set.size);+ sched->tensor_copies = calloc(sizeof(sched->tensor_copies[0]), sched->hash_set.size);+ sched->node_backend_ids = calloc(sizeof(sched->node_backend_ids[0]), graph_size);sched->n_backends = n_backends;for (int i = 0; i < n_backends; i++) {@@ -1418,14 +1540,9 @@ ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backsched->bufts[i] = bufts ? bufts[i] : ggml_backend_get_default_buffer_type(backends[i]);}- sched->galloc = ggml_gallocr_new();-- // init measure allocs for each backend- for (int i = 0; i < n_backends; i++) {- sched->tallocs[i] = ggml_tallocr_new_measure_from_buft(sched->bufts[i]);- }+ sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends);- sched_reset(sched);+ ggml_backend_sched_reset(sched);return sched;}@@ -1434,49 +1551,54 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {if (sched == NULL) {return;}- for (int i = 0; i < sched->n_backends; i++) {- ggml_tallocr_free(sched->tallocs[i]);- }ggml_gallocr_free(sched->galloc);ggml_free(sched->ctx);free(sched->hash_set.keys);- free(sched->node_talloc);- free(sched->node_copies);+ free(sched->tensor_backend_id);+ free(sched->tensor_copies);+ free(sched->node_backend_ids);free(sched);}-void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {- GGML_ASSERT(ggml_tallocr_is_measure(sched->tallocs[0])); // can only be initialized once+void ggml_backend_sched_reset(ggml_backend_sched_t sched) {+ // reset state for the next run+ size_t hash_size = sched->hash_set.size;+ memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size); // NOLINT+ memset(sched->tensor_backend_id, -1, sizeof(sched->tensor_backend_id[0]) * hash_size);+ memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size);++ sched->is_reset = true;+}- sched_split_graph(sched, measure_graph);- sched_alloc_splits(sched);+bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {+ ggml_backend_sched_split_graph(sched, measure_graph);- // allocate buffers and reset allocators- for (int i = 0; i < sched->n_backends; i++) {- size_t size = ggml_tallocr_max_size(sched->tallocs[i]);- ggml_tallocr_free(sched->tallocs[i]);- sched->tallocs[i] = ggml_tallocr_new_from_buft(sched->bufts[i], size);+ if (!ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids)) {+ return false;}- sched_reset(sched);+ ggml_backend_sched_reset(sched);+ return true;}-void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);if (!sched->is_reset) {- sched_reset(sched);+ ggml_backend_sched_reset(sched);}- sched_split_graph(sched, graph);- sched_alloc_splits(sched);- sched_compute_splits(sched);-}+ ggml_backend_sched_split_graph(sched, graph);+ if (!ggml_backend_sched_alloc_splits(sched)) {+ return false;+ }-void ggml_backend_sched_reset(ggml_backend_sched_t sched) {- sched_reset(sched);-}+ if (!ggml_backend_sched_compute_splits(sched)) {+ return false;+ }+ return true;+}void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) {sched->callback_eval = callback;@@ -1487,37 +1609,30 @@ int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {return sched->n_splits;}-ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend) {- int backend_index = sched_backend_prio(sched, backend);- GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);- return sched->tallocs[backend_index];-}--ggml_backend_buffer_t ggml_backend_sched_get_buffer(ggml_backend_sched_t sched, ggml_backend_t backend) {- int backend_index = sched_backend_prio(sched, backend);+size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) {+ int backend_index = ggml_backend_sched_backend_id(sched, backend);GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);- return ggml_tallocr_get_buffer(sched->tallocs[backend_index]);+ return ggml_gallocr_get_buffer_size(sched->galloc, backend_index);}void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {- int backend_index = sched_backend_prio(sched, backend);+ int backend_index = ggml_backend_sched_backend_id(sched, backend);GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);- node_allocr(node) = sched->tallocs[backend_index];+ tensor_backend_id(node) = backend_index;}ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {- ggml_tallocr_t allocr = node_allocr(node);- if (allocr == NULL) {+ int backend_index = tensor_backend_id(node);+ if (backend_index == -1) {return NULL;}- return get_allocr_backend(sched, allocr);+ return sched->backends[backend_index];}// utilsvoid ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {GGML_ASSERT(tensor->buffer == NULL);- //GGML_ASSERT(tensor->data == NULL); // views of pre-allocated tensors may have the data set in ggml_new_tensor, but still need to be initialized by the backendGGML_ASSERT(tensor->view_src != NULL);GGML_ASSERT(tensor->view_src->buffer != NULL);GGML_ASSERT(tensor->view_src->data != NULL);@@ -1541,7 +1656,7 @@ void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensorggml_backend_buffer_init_tensor(buffer, tensor);}-static struct ggml_tensor * graph_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies,+static struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies,struct ggml_context * ctx_allocated, struct ggml_context * ctx_unallocated, struct ggml_tensor * src) {GGML_ASSERT(src != NULL);@@ -1554,7 +1669,7 @@ static struct ggml_tensor * graph_dup_tensor(struct ggml_hash_set hash_set, strustruct ggml_tensor * dst = ggml_dup_tensor_layout(src->data && !src->view_src ? ctx_allocated : ctx_unallocated, src);if (src->view_src != NULL) {- dst->view_src = graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src);+ dst->view_src = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src);dst->view_offs = src->view_offs;}dst->op = src->op;@@ -1565,16 +1680,16 @@ static struct ggml_tensor * graph_dup_tensor(struct ggml_hash_set hash_set, strufor (int i = 0; i < GGML_MAX_SRC; i++) {struct ggml_tensor * s = src->src[i];if (s == NULL) {- break;+ continue;}- dst->src[i] = graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s);+ dst->src[i] = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s);}node_copies[id] = dst;return dst;}-static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) {+static void graph_copy_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) {size_t id = ggml_hash_find(hash_set, src);if (node_init[id]) {return;@@ -1583,7 +1698,7 @@ static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensorstruct ggml_tensor * dst = node_copies[id];if (dst->view_src != NULL) {- graph_init_tensor(hash_set, node_copies, node_init, src->view_src);+ graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src);ggml_backend_view_init(dst->view_src->buffer, dst);}else {@@ -1594,19 +1709,19 @@ static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensorfor (int i = 0; i < GGML_MAX_SRC; i++) {struct ggml_tensor * s = src->src[i];if (s == NULL) {- break;+ continue;}- graph_init_tensor(hash_set, node_copies, node_init, s);+ graph_copy_init_tensor(hash_set, node_copies, node_init, s);}}struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph) {struct ggml_hash_set hash_set = {/* .size = */ graph->visited_hash_table.size,- /* .keys = */ calloc(sizeof(hash_set.keys[0]) * graph->visited_hash_table.size, 1)+ /* .keys = */ calloc(sizeof(hash_set.keys[0]), graph->visited_hash_table.size) // NOLINT};- struct ggml_tensor ** node_copies = calloc(sizeof(node_copies[0]) * hash_set.size, 1);- bool * node_init = calloc(sizeof(node_init[0]) * hash_set.size, 1);+ struct ggml_tensor ** node_copies = calloc(sizeof(node_copies[0]), hash_set.size); // NOLINT+ bool * node_init = calloc(sizeof(node_init[0]), hash_set.size);struct ggml_init_params params = {/* .mem_size = */ ggml_tensor_overhead()*hash_set.size + ggml_graph_overhead_custom(graph->size, false),@@ -1635,7 +1750,7 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s// dup nodesfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- graph_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node);+ graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node);}// allocate nodes@@ -1660,7 +1775,7 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s// copy data and init viewsfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- graph_init_tensor(hash_set, node_copies, node_init, node);+ graph_copy_init_tensor(hash_set, node_copies, node_init, node);}// build graph copy@@ -1748,7 +1863,6 @@ static const struct ggml_backend_api kGgmlBackendApi = {system_malloc,ggml_backend_register,ggml_backend_buffer_init,- ggml_backend_buffer_is_host,ggml_backend_cpu_buffer_from_ptr,ggml_backend_cpu_buffer_type,ggml_backend_buft_get_alloc_size,@@ -1772,6 +1886,7 @@ static const struct ggml_backend_api kGgmlBackendApi = {ggml_row_size,ggml_rope_yarn_corr_dims,ggml_op_desc,+ ggml_backend_buffer_is_host,};const struct ggml_backend_api *ggml_backend_api(void) {diff --git a/llama.cpp/ggml-backend.h b/llama.cpp/ggml-backend.hindex 2ca1654e..9ea70ead 100644--- a/llama.cpp/ggml-backend.h+++ b/llama.cpp/ggml-backend.h@@ -20,6 +20,7 @@ extern "C" {GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);+ GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);GGML_API GGML_CALL size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);@@ -36,6 +37,7 @@ extern "C" {GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);GGML_API GGML_CALL void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);+ GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);GGML_API GGML_CALL bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);@@ -54,6 +56,7 @@ extern "C" {GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);+ GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);@@ -80,8 +83,9 @@ extern "C" {GGML_API ggml_backend_t ggml_backend_cpu_init(void);- GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend);- GGML_API void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);+ GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend);+ GGML_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);+ GGML_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);// Create a backend buffer from an existing pointerGGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);@@ -126,11 +130,7 @@ extern "C" {// in build_graph:build_graph(...) {- // allocating tensors in a specific backend (optional, recommended: pre-allocate inputs in a different buffer)- alloc_cpu = ggml_backend_sched_get_allocr(sched, backend_cpu);- ggml_allocr_alloc(alloc_cpu, tensor);-- // manually assigning nodes to a backend (optional, shouldn't be needed in most cases)+ // manually assign nodes to a backend (optional, should not be needed in most cases)struct ggml_tensor * node = ggml_mul_mat(ctx, ...);ggml_backend_sched_set_node_backend(sched, node, backend_gpu);}@@ -160,20 +160,19 @@ extern "C" {GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size);GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);// Initialize backend buffers from a measure graph- GGML_API void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);+ GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);// Get the number of splits of the last graphGGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);- GGML_API ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend);- GGML_API ggml_backend_buffer_t ggml_backend_sched_get_buffer (ggml_backend_sched_t sched, ggml_backend_t backend);+ GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);// Allocate and compute graph on the backend scheduler- GGML_API void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);+ GGML_API bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);- // Reset all assignments and allocators - must be called before using the sched allocators to allocate inputs+ // Reset all assignments and allocators - must be called before changing the node backendsGGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);// Set a callback to be called for each resulting node during graph computediff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex f9e7d0bf..4a049e73 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -61,6 +61,8 @@#define cudaDeviceProp hipDeviceProp_t#define cudaDeviceSynchronize hipDeviceSynchronize#define cudaError_t hipError_t+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled+#define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled#define cudaEventCreateWithFlags hipEventCreateWithFlags#define cudaEventDisableTiming hipEventDisableTiming#define cudaEventRecord hipEventRecord@@ -174,6 +176,8 @@#define cudaDeviceProp hipDeviceProp_t#define cudaDeviceSynchronize hipDeviceSynchronize#define cudaError_t hipError_t+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled+#define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled#define cudaEventCreateWithFlags hipEventCreateWithFlags#define cudaEventDisableTiming hipEventDisableTiming#define cudaEventRecord hipEventRecord@@ -320,8 +324,8 @@ GGML_CALL void ggml_cuda_link(const struct ggml_backend_api *backend_api) {#define CUDA_USE_TENSOR_CORES#endif-// max batch size to use MMQ kernels when tensor cores are available-#define MMQ_MAX_BATCH_SIZE 32+#define MMVQ_MAX_BATCH_SIZE 8 // max batch size to use MMVQ kernels+#define MMQ_MAX_BATCH_SIZE 32 // max batch size to use MMQ kernels when tensor cores are available#if defined(GGML_USE_HIPBLAS)#define __CUDA_ARCH__ 1300@@ -361,6 +365,10 @@ static __device__ __forceinline__ int __vsubss4(const int a, const int b) {#endif // __has_builtin(__builtin_elementwise_sub_sat)}+static __device__ __forceinline__ int __vsub4(const int a, const int b) {+ return __vsubss4(a, b);+}+static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx1030__)c = __builtin_amdgcn_sdot4(a, b, c, false);@@ -675,6 +683,32 @@ typedef struct {} block_iq2_xs;static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");+#define QR3_XXS 8+#define QI3_XXS (QK_K / (4*QR3_XXS))+typedef struct {+ half d;+ uint8_t qs[3*(QK_K/8)];+} block_iq3_xxs;+static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");++#define QR1_S 8+#define QI1_S (QK_K / (4*QR1_S))+typedef struct {+ half d;+ uint8_t qs[QK_K/8];+ uint8_t scales[QK_K/16];+} block_iq1_s;+static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");++#define QK4_NL 32+#define QR4_NL 2+#define QI4_NL (QK4_NL / (4*QR4_NL))+typedef struct {+ half d;+ uint8_t qs[QK4_NL/2];+} block_iq4_nl;+static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding");+#define WARP_SIZE 32#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses@@ -682,6 +716,8 @@ static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16#define CUDA_SILU_BLOCK_SIZE 256#define CUDA_TANH_BLOCK_SIZE 256#define CUDA_RELU_BLOCK_SIZE 256+#define CUDA_HARDSIGMOID_BLOCK_SIZE 256+#define CUDA_HARDSWISH_BLOCK_SIZE 256#define CUDA_SQR_BLOCK_SIZE 256#define CUDA_CPY_BLOCK_SIZE 32#define CUDA_SCALE_BLOCK_SIZE 256@@ -698,6 +734,7 @@ static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16#define CUDA_PAD_BLOCK_SIZE 256#define CUDA_ACC_BLOCK_SIZE 256#define CUDA_IM2COL_BLOCK_SIZE 256+#define CUDA_POOL2D_BLOCK_SIZE 256#define CUDA_Q8_0_NE_ALIGN 2048@@ -799,18 +836,18 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {return a;}-static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));- }- return a;-#else- (void) a;- NO_DEVICE_CODE;-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL-}+//static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {+//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+//#pragma unroll+// for (int mask = 16; mask > 0; mask >>= 1) {+// a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));+// }+// return a;+//#else+// (void) a;+// NO_DEVICE_CODE;+//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+//}static __device__ __forceinline__ float warp_reduce_max(float x) {#pragma unroll@@ -820,18 +857,18 @@ static __device__ __forceinline__ float warp_reduce_max(float x) {return x;}-static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));- }- return x;-#else- (void) x;- NO_DEVICE_CODE;-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX-}+//static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {+//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX+//#pragma unroll+// for (int mask = 16; mask > 0; mask >>= 1) {+// x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));+// }+// return x;+//#else+// (void) x;+// NO_DEVICE_CODE;+//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX+//}static __device__ __forceinline__ float op_repeat(const float a, const float b) {return b;@@ -983,6 +1020,24 @@ static __global__ void relu_f32(const float * x, float * dst, const int k) {dst[i] = fmaxf(x[i], 0);}+static __global__ void hardsigmoid_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;+ }+ dst[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));+}++static __global__ void hardswish_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;+ }+ dst[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));+}+static __global__ void leaky_relu_f32(const float * x, float * dst, const int k, const float negative_slope) {const int i = blockDim.x*blockIdx.x + threadIdx.x;if (i >= k) {@@ -1785,6 +1840,172 @@ static const __device__ uint64_t iq2xs_grid[512] = {0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,};+static const __device__ uint32_t iq3xxs_grid[256] = {+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,+};++static const __device__ uint64_t iq1s_grid[512] = {+ 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,+ 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,+ 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,+ 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,+ 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,+ 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,+ 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,+ 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,+ 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,+ 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,+ 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,+ 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,+ 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,+ 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,+ 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,+ 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,+ 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,+ 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,+ 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,+ 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,+ 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,+ 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,+ 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,+ 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,+ 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,+ 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,+ 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,+ 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,+ 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,+ 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,+ 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,+ 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,+ 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,+ 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,+ 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,+ 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,+ 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,+ 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,+ 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,+ 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,+ 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,+ 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,+ 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,+ 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,+ 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,+ 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,+ 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,+ 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,+ 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,+ 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,+ 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,+ 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,+ 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,+ 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,+ 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,+ 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,+ 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,+ 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,+ 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,+ 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,+ 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,+ 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,+ 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,+ 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,+ 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,+ 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,+ 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,+ 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,+ 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,+ 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,+ 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,+ 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,+ 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,+ 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,+ 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,+ 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,+ 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,+ 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,+ 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,+ 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,+ 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,+ 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,+ 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,+ 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,+ 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,+ 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,+ 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,+ 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,+ 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,+ 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,+ 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,+ 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,+ 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,+ 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,+ 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,+ 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,+ 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,+ 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,+ 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,+ 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,+ 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,+ 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,+ 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,+ 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,+ 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,+ 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,+ 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,+ 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,+ 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,+ 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,+ 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,+ 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,+ 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,+ 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,+ 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,+ 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,+ 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,+ 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,+ 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,+ 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,+ 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,+ 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,+ 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,+ 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,+ 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,+ 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,+};+static const __device__ uint8_t ksigns_iq2xs[128] = {0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,@@ -1796,6 +2017,43 @@ static const __device__ uint8_t ksigns_iq2xs[128] = {240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,};+//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+static const __device__ uint64_t ksigns64[128] = {+ 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,+ 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,+ 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,+ 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,+ 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,+ 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,+ 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,+ 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,+ 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,+ 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,+ 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,+ 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,+ 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,+ 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,+ 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,+ 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,+ 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,+ 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,+ 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,+ 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,+ 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,+ 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,+ 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,+ 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,+ 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,+ 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,+ 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,+ 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,+ 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,+ 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,+ 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,+ 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,+};+//#endif+static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};inline bool ggml_cuda_supports_mmq(enum ggml_type type) {@@ -1862,6 +2120,77 @@ static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst}+template<typename dst_t>+static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {++ const int i = blockIdx.x;+ const block_iq3_xxs * x = (const block_iq3_xxs *) vx;++ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint8_t * q3 = x[i].qs + 8*ib;+ const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);+ const uint32_t aux32 = gas[0] | (gas[1] << 16);+ const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];+ for (int j = 0; j < 4; ++j) {+ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+#else+ assert(false);+#endif++}++template<typename dst_t>+static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {++ const int i = blockIdx.x;+ const block_iq1_s * x = (const block_iq1_s *) vx;++ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const int i8 = 4*ib+il;+ uint8_t h = x[i].scales[i8/2] >> 4*(i8%2);+ const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5)));+ const float d = (float)x[i].d * (2*(h & 7) + 1);+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j];+#else+ assert(false);+#endif++}++static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};++template<typename dst_t>+static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {++ const int i = blockIdx.x;+ const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);++ const int tid = threadIdx.x;+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;+ const uint8_t * q4 = x[ib].qs + 4*il;+ const float d = (float)x[ib].d;+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];+ y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];+ }++}+static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");@@ -4485,6 +4814,7 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics#if QK_K == 256const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;@@ -4495,30 +4825,166 @@ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(const uint8_t ls2 = bq2->scales[ib32] >> 4;int sumi1 = 0;for (int l = 0; l < 2; ++l) {- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];- for (int j = 0; j < 8; ++j) {- sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));+ const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);+ sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);+ sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);q8 += 8;}int sumi2 = 0;for (int l = 2; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];- for (int j = 0; j < 8; ++j) {- sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));+ const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);+ sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);+ sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);q8 += 8;}const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);+#else+ (void) ksigns64;+ assert(false);+ return 0.f;+#endif+#else+ (void) ksigns64;+ assert(false);+ return 0.f;+#endif+}++static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;++ const int ib32 = iqs;+ const uint8_t * q3 = bq2->qs + 8*ib32;+ const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ uint32_t aux32 = gas[0] | (gas[1] << 16);+ int sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];+ const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));+ const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);+ sumi = __dp4a(grid_l, *((int *)q8+0), sumi);+ sumi = __dp4a(grid_h, *((int *)q8+1), sumi);+ q8 += 8;+ aux32 >>= 7;+ }+ const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;+ return d * sumi;+#else+ assert(false);+ return 0.f;+#endif+#else+ assert(false);+ return 0.f;+#endif+}++static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if QK_K == 256+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq;++ const int ib32 = iqs;+ int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0;+ const uint8_t h1 = bq1->scales[2*ib32+0];+ const uint8_t h2 = bq1->scales[2*ib32+1];+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const int * q8 = (const int *)bq8_1[ib32].qs;+ const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5)));+ const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1)));+ const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5)));+ const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1)));+ for (int j = 0; j < 2; ++j) {+ sumi1 = __dp4a(q8[j+0], grid1[j], sumi1);+ sumi2 = __dp4a(q8[j+2], grid2[j], sumi2);+ sumi3 = __dp4a(q8[j+4], grid3[j], sumi3);+ sumi4 = __dp4a(q8[j+6], grid4[j], sumi4);+ }+#else+ const int8_t * q8 = bq8_1[ib32].qs;+ const int8_t * grid1 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5)));+ const int8_t * grid2 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1)));+ const int8_t * grid3 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5)));+ const int8_t * grid4 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1)));+ for (int j = 0; j < 8; ++j) {+ sumi1 += q8[j+ 0] * grid1[j];+ sumi2 += q8[j+ 8] * grid2[j];+ sumi3 += q8[j+16] * grid3[j];+ sumi4 += q8[j+24] * grid4[j];+ }+#endif+ const float d = (float)bq1->d * __low2float(bq8_1[ib32].ds);+ return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) ++ sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1));#elseassert(false);return 0.f;#endif}+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,+ int & val1, int & val2) {++ uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;+ aux32 = q4 & 0x0f0f0f0f;+ uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);+ uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);+ val1 = v1 | (v2 << 16);+ aux32 = (q4 >> 4) & 0x0f0f0f0f;+ v1 = values[q8[0]] | (values[q8[1]] << 8);+ v2 = values[q8[2]] | (values[q8[3]] << 8);+ val2 = v1 | (v2 << 16);+}+#endif++static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {++ const block_iq4_nl * bq = (const block_iq4_nl *) vbq;++#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;+ const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;++ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;++ int v1, v2;+ int sumi1 = 0, sumi2 = 0;+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {+ const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);+ get_int_from_table_16(aux, values, v1, v2);+ sumi1 = __dp4a(v1, q8[l+0], sumi1);+ sumi2 = __dp4a(v2, q8[l+4], sumi2);+ }++#else+ const uint8_t * q4 = bq->qs + 4*iqs;+ const int8_t * q8 = bq8_1->qs + 4*iqs;++ int sumi1 = 0, sumi2 = 0;+ for (int l = 0; l < 4*VDR_Q4_0_Q8_1_MMVQ; ++l) {+ sumi1 += q8[l+ 0] * kvalues_iq4nl[q4[l] & 0xf];+ sumi2 += q8[l+16] * kvalues_iq4nl[q4[l] >> 4];+ }+#endif+ const float d = (float)bq->d * __low2float(bq8_1->ds);+ return d * (sumi1 + sumi2);+}+template <int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>static __device__ __forceinline__ void mul_mat_q(@@ -5307,41 +5773,81 @@ template <bool need_check> static __global__ void#endif // __CUDA_ARCH__ >= CC_VOLTA}-template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>-static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows) {- const int row = blockIdx.x*blockDim.y + threadIdx.y;+template <int ncols_y, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+// tell the compiler to use as many registers as it wants, see nwarps definition below+__launch_bounds__((ncols_y <= 4 ? 4 : 2)*WARP_SIZE, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void mul_mat_vec_q(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int nrows_y, const int nrows_dst) {- if (row >= nrows) {- return;- }+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3))+ constexpr int nwarps = 1;+ constexpr int rows_per_cuda_block = 1;+#else+ constexpr int nwarps = ncols_y <= 4 ? 4 : 2;+ constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2;+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3)- const int blocks_per_row = ncols / qk;- const int blocks_per_warp = vdr * WARP_SIZE / qi;+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;+ const int row0 = rows_per_cuda_block*blockIdx.x;+ const int blocks_per_row_x = ncols_x / qk;+ const int blocks_per_col_y = nrows_y / QK8_1;+ constexpr int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi;// partial sum for each thread- float tmp = 0.0f;+ float tmp[ncols_y][rows_per_cuda_block] = {0.0f};const block_q_t * x = (const block_q_t *) vx;const block_q8_1 * y = (const block_q8_1 *) vy;- for (int i = threadIdx.x / (qi/vdr); i < blocks_per_row; i += blocks_per_warp) {- const int ibx = row*blocks_per_row + i; // x block index+ for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {+ const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx- const int iby = i * (qk/QK8_1); // y block index that aligns with ibx+ // x block quant index when casting the quants to int+ const int kqs = vdr * (tid % (qi/vdr));- const int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int+#pragma unroll+ for (int j = 0; j < ncols_y; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+ tmp[j][i] += vec_dot_q_cuda(+ &x[kbx + (row0 + i)*blocks_per_row_x], &y[j*blocks_per_col_y + kby], kqs);+ }+ }+ }- tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);+ __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y][rows_per_cuda_block][WARP_SIZE];+ if (threadIdx.y > 0) {+#pragma unroll+ for (int j = 0; j < ncols_y; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+ tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];+ }+ }+ }+ __syncthreads();+ if (threadIdx.y > 0) {+ return;}// sum up partial sums and write back result#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- tmp += __shfl_xor_sync(0xffffffff, tmp, mask, 32);- }+ for (int j = 0; j < ncols_y; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+#pragma unroll+ for (int l = 0; l < nwarps-1; ++l) {+ tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];+ }+ tmp[j][i] = warp_reduce_sum(tmp[j][i]);+ }- if (threadIdx.x == 0) {- dst[row] = tmp;+ if (threadIdx.x < rows_per_cuda_block) {+ dst[j*nrows_dst + row0 + threadIdx.x] = tmp[j][threadIdx.x];+ }}}@@ -5529,27 +6035,37 @@ static __device__ void cpy_1_f16_f16(const char * cxi, char * cdsti) {*dsti = *xi;}+static __device__ void cpy_1_f16_f32(const char * cxi, char * cdsti) {+ const half * xi = (const half *) cxi;+ float * dsti = (float *) cdsti;++ *dsti = *xi;+}+template <cpy_kernel_t cpy_1>static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,+ const int nb12, const int nb13) {const int i = blockDim.x*blockIdx.x + threadIdx.x;if (i >= ne) {return;}- // determine indices i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor+ // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor// then combine those indices with the corresponding byte offsets to get the total offsets- const int i02 = i / (ne00*ne01);- const int i01 = (i - i02*ne01*ne00) / ne00;- const int i00 = i - i02*ne01*ne00 - i01*ne00;- const int x_offset = i00*nb00 + i01*nb01 + i02*nb02;-- const int i12 = i / (ne10*ne11);- const int i11 = (i - i12*ne10*ne11) / ne10;- const int i10 = i - i12*ne10*ne11 - i11*ne10;- const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12;+ const int i03 = i/(ne00 * ne01 * ne02);+ const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);+ const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;+ const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;+ const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;++ const int i13 = i/(ne10 * ne11 * ne12);+ const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);+ const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;+ const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;+ const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;cpy_1(cx + x_offset, cdst + dst_offset);}@@ -5643,23 +6159,26 @@ static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {template <cpy_kernel_t cpy_blck, int qk>static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,+ const int nb12, const int nb13) {const int i = (blockDim.x*blockIdx.x + threadIdx.x)*qk;if (i >= ne) {return;}- const int i02 = i / (ne00*ne01);- const int i01 = (i - i02*ne01*ne00) / ne00;- const int i00 = (i - i02*ne01*ne00 - i01*ne00);- const int x_offset = i00*nb00 + i01*nb01 + i02*nb02;+ const int i03 = i/(ne00 * ne01 * ne02);+ const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);+ const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;+ const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;+ const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;- const int i12 = i / (ne10*ne11);- const int i11 = (i - i12*ne10*ne11) / ne10;- const int i10 = (i - i12*ne10*ne11 - i11*ne10)/qk;- const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12;+ const int i13 = i/(ne10 * ne11 * ne12);+ const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);+ const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;+ const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;+ const int dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;cpy_blck(cx + x_offset, cdst + dst_offset);}@@ -5828,7 +6347,7 @@ static __global__ void alibi_f32(const float * x, float * dst, const int ncols,}static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {- const int row = blockIdx.y;+ const int row = blockIdx.x;const int col = threadIdx.x;float sum = 0.0f;@@ -5900,53 +6419,53 @@ static __global__ void diag_mask_inf_f32(const float * x, float * dst, const intdst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;}-template <bool vals_smem, int ncols_template, int block_size_template, bool need_check>-static __global__ void soft_max_f16(const float * x, const float * y, float * dst, const int ncols_par, const int nrows_y, const float scale) {-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX- const int ncols_data = ncols_template == 0 ? ncols_par : ncols_template;- const int ncols_smem = GGML_PAD(ncols_data, 2*WARP_SIZE)/2;+template <bool vals_smem, int ncols_template, int block_size_template>+static __global__ void soft_max_f32(const float * x, const float * mask, const float * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {+ const int ncols = ncols_template == 0 ? ncols_par : ncols_template;const int tid = threadIdx.x;const int rowx = blockIdx.x;- const int rowy = rowx % nrows_y; // broadcast the mask (y) in the row dimension+ const int rowy = rowx % nrows_y; // broadcast the mask in the row dimensionconst int block_size = block_size_template == 0 ? blockDim.x : block_size_template;const int warp_id = threadIdx.x / WARP_SIZE;const int lane_id = threadIdx.x % WARP_SIZE;- extern __shared__ half data_soft_max_f16[];- half * buf_iw = data_soft_max_f16 + 0; // shared memory buffer for inter-warp communication- // (shared memory) buffer to cache values between iterations:- half2 * vals = vals_smem ? (half2 *) (buf_iw + WARP_SIZE) : (half2 *) (dst + rowx*ncols_data);- // if the buffer is larger than max. shared memory per block, use dst as temp. buffer instead- // in that case col_smem == col_data must be enforced to avoid race conditions+ float slope = 0.0f;- half2 max_val = make_half2(-INFINITY, -INFINITY);+ // ALiBi+ if (max_bias > 0.0f) {+ const int h = rowx/nrows_y; // head index-#pragma unroll- for (int col0 = 0; col0 < ncols_smem; col0 += block_size) {- const int col_data = 2*col0 + 2*WARP_SIZE*warp_id + lane_id;- const int col_smem = vals_smem ? col0 + tid : col_data;+ const float base = h < n_head_log2 ? m0 : m1;+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;- const int ix = rowx*ncols_data + col_data;- const int iy = rowy*ncols_data + col_data;+ slope = powf(base, exp);+ }- half2 val;- if (need_check && col_data + 0 >= ncols_data) {- val.x = -INFINITY;- } else {- val.x = x[ix + 0]*scale + (y ? y[iy + 0] : 0.0f);- }- if (need_check && col_data + WARP_SIZE >= ncols_data) {- val.y = -INFINITY;- } else {- val.y = x[ix + WARP_SIZE]*scale + (y ? y[iy + WARP_SIZE] : 0.0f);- }- if (!need_check || col_smem < (vals_smem ? ncols_smem : ncols_data)) {- vals[col_smem] = val;+ extern __shared__ float data_soft_max_f32[];+ float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication+ // shared memory buffer to cache values between iterations:+ float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + rowx*ncols;++ float max_val = -INFINITY;++#pragma unroll+ for (int col0 = 0; col0 < ncols; col0 += block_size) {+ const int col = col0 + tid;++ if (ncols_template == 0 && col >= ncols) {+ break;}- max_val = __hmax2(max_val, val);++ const int ix = rowx*ncols + col;+ const int iy = rowy*ncols + col;++ const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + (pos ? slope*pos[col] : 0.0f);++ vals[col] = val;+ max_val = max(max_val, val);}// find the max value in the block@@ -5958,30 +6477,27 @@ static __global__ void soft_max_f16(const float * x, const float * y, float * ds__syncthreads();if (lane_id == 0) {- buf_iw[warp_id] = __hmax(max_val.x, max_val.y);+ buf_iw[warp_id] = max_val;}__syncthreads();- max_val = __half2half2(buf_iw[lane_id]);+ max_val = buf_iw[lane_id];max_val = warp_reduce_max(max_val);- } else {- max_val = __half2half2(__hmax(max_val.x, max_val.y));}- half2 tmp = make_half2(0.0f, 0.0f); // partial sums+ float tmp = 0.0f; // partial sum#pragma unroll- for (int col0 = 0; col0 < ncols_smem; col0 += block_size) {- const int col_smem = vals_smem ? col0 + tid : 2*col0 + 2*warp_id*WARP_SIZE + lane_id;+ for (int col0 = 0; col0 < ncols; col0 += block_size) {+ const int col = col0 + tid;- if (ncols_template == 0 && col_smem >= (vals_smem ? ncols_smem : ncols_data)) {+ if (ncols_template == 0 && col >= ncols) {break;}- const half2 val = h2exp(vals[col_smem] - max_val);-+ const float val = expf(vals[col] - max_val);tmp += val;- vals[col_smem] = val;+ vals[col] = val;}// find the sum of exps in the block@@ -5993,145 +6509,31 @@ static __global__ void soft_max_f16(const float * x, const float * y, float * ds__syncthreads();if (lane_id == 0) {- buf_iw[warp_id] = tmp.x + tmp.y;+ buf_iw[warp_id] = tmp;}__syncthreads();- tmp = __half2half2(buf_iw[lane_id]);+ tmp = buf_iw[lane_id];tmp = warp_reduce_sum(tmp);- } else {- tmp = __half2half2(tmp.x + tmp.y);}- const half2 inv_sum = make_half2(1.0f, 1.0f) / tmp;+ const float inv_sum = 1.0f / tmp;#pragma unroll- for (int col0 = 0; col0 < ncols_smem; col0 += block_size) {- const int col_data = 2*col0 + 2*WARP_SIZE*warp_id + lane_id;- const int col_smem = vals_smem ? col0 + tid : col_data;-- const int idst = rowx*ncols_data + col_data;- const half2 result = vals[col_smem] * inv_sum;-- if (need_check && col_data + 0 >= ncols_data) {- return;- }- dst[idst] = result.x;+ for (int col0 = 0; col0 < ncols; col0 += block_size) {+ const int col = col0 + tid;- if (need_check && col_data + WARP_SIZE >= ncols_data) {+ if (ncols_template == 0 && col >= ncols) {return;}- dst[idst + WARP_SIZE] = result.y;+ const int idst = rowx*ncols + col;+ dst[idst] = vals[col] * inv_sum;}-#else- (void) x; (void) y; (void) dst; (void) ncols_par; (void) nrows_y; (void) scale;- NO_DEVICE_CODE;-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX}-template <bool vals_smem, int ncols_template, int block_size_template>-static __global__ void soft_max_f32(const float * x, const float * y, float * dst, const int ncols_par, const int nrows_y, const float scale) {- const int ncols = ncols_template == 0 ? ncols_par : ncols_template;-- const int tid = threadIdx.x;- const int rowx = blockIdx.x;- const int rowy = rowx % nrows_y; // broadcast the mask (y) in the row dimension-- const int block_size = block_size_template == 0 ? blockDim.x : block_size_template;-- const int warp_id = threadIdx.x / WARP_SIZE;- const int lane_id = threadIdx.x % WARP_SIZE;-- extern __shared__ float data_soft_max_f32[];- float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication- // shared memory buffer to cache values between iterations:- float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + rowx*ncols;-- float max_val = -INFINITY;--#pragma unroll- for (int col0 = 0; col0 < ncols; col0 += block_size) {- const int col = col0 + tid;-- if (ncols_template == 0 && col >= ncols) {- break;- }-- const int ix = rowx*ncols + col;- const int iy = rowy*ncols + col;-- const float val = x[ix]*scale + (y ? y[iy] : 0.0f);- vals[col] = val;- max_val = max(max_val, val);- }-- // find the max value in the block- max_val = warp_reduce_max(max_val);- if (block_size > WARP_SIZE) {- if (warp_id == 0) {- buf_iw[lane_id] = -INFINITY;- }- __syncthreads();-- if (lane_id == 0) {- buf_iw[warp_id] = max_val;- }- __syncthreads();-- max_val = buf_iw[lane_id];- max_val = warp_reduce_max(max_val);- }-- float tmp = 0.0f; // partial sum--#pragma unroll- for (int col0 = 0; col0 < ncols; col0 += block_size) {- const int col = col0 + tid;-- if (ncols_template == 0 && col >= ncols) {- break;- }-- const float val = expf(vals[col] - max_val);- tmp += val;- vals[col] = val;- }-- // find the sum of exps in the block- tmp = warp_reduce_sum(tmp);- if (block_size > WARP_SIZE) {- if (warp_id == 0) {- buf_iw[lane_id] = 0.0f;- }- __syncthreads();-- if (lane_id == 0) {- buf_iw[warp_id] = tmp;- }- __syncthreads();-- tmp = buf_iw[lane_id];- tmp = warp_reduce_sum(tmp);- }-- const float inv_sum = 1.0f / tmp;--#pragma unroll- for (int col0 = 0; col0 < ncols; col0 += block_size) {- const int col = col0 + tid;-- if (ncols_template == 0 && col >= ncols) {- return;- }-- const int idst = rowx*ncols + col;- dst[idst] = vals[col] * inv_sum;- }-}--static __global__ void scale_f32(const float * x, float * dst, const float scale, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+static __global__ void scale_f32(const float * x, float * dst, const float scale, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;if (i >= k) {return;@@ -6150,9 +6552,10 @@ static __global__ void clamp_f32(const float * x, float * dst, const float min,dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]);}-static __global__ void im2col_f32_f16(- const float * x, half * dst,- int offset_delta, int IW, int IH, int OW, int KW, int KH, int pelements, int CHW,+template <typename T>+static __global__ void im2col_kernel(+ const float * x, T * dst, int batch_offset,+ int offset_delta, int IC, int IW, int IH, int OH, int OW, int KW, int KH, int pelements, int CHW,int s0, int s1, int p0, int p1, int d0, int d1) {const int i = threadIdx.x + blockIdx.x * blockDim.x;if (i >= pelements) {@@ -6165,21 +6568,73 @@ static __global__ void im2col_f32_f16(const int ky = (i - kd) / OW;const int ix = i % OW;+ const int oh = blockIdx.y;+ const int batch = blockIdx.z / IC;+ const int ic = blockIdx.z % IC;+const int64_t iiw = ix * s0 + kx * d0 - p0;- const int64_t iih = blockIdx.y * s1 + ky * d1 - p1;+ const int64_t iih = oh * s1 + ky * d1 - p1;const int64_t offset_dst =- (blockIdx.y * OW + ix) * CHW +- (blockIdx.z * (KW * KH) + ky * KW + kx);+ ((batch * OH + oh) * OW + ix) * CHW ++ (ic * (KW * KH) + ky * KW + kx);if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {- dst[offset_dst] = __float2half(0.0f);+ dst[offset_dst] = 0.0f;} else {- const int64_t offset_src = blockIdx.z * offset_delta;- dst[offset_dst] = __float2half(x[offset_src + iih * IW + iiw]);+ const int64_t offset_src = ic * offset_delta + batch * batch_offset;+ dst[offset_dst] = x[offset_src + iih * IW + iiw];}}+template <typename Ti, typename To>+static __global__ void pool2d_nchw_kernel(+ const int ih, const int iw, const int oh, const int ow,+ const int kh, const int kw, const int sh, const int sw,+ const int ph, const int pw, const int parallel_elements,+ const Ti* src, To* dst, const enum ggml_op_pool op) {+ int idx = threadIdx.x + blockIdx.x * blockDim.x;+ if (idx >= parallel_elements) {+ return;+ }++ const int I_HW = ih * iw;+ const int O_HW = oh * ow;+ const int nc = idx / O_HW;+ const int cur_oh = idx % O_HW / ow;+ const int cur_ow = idx % O_HW % ow;+ const Ti* i_ptr = src + nc * I_HW;+ To* o_ptr = dst + nc * O_HW;+ const int start_h = cur_oh * sh - ph;+ const int bh = max(0, start_h);+ const int eh = min(ih, start_h + kh);+ const int start_w = cur_ow * sw - pw;+ const int bw = max(0, start_w);+ const int ew = min(iw, start_w + kw);+ const To scale = 1. / (kh * kw);+ To res = 0;++ switch (op) {+ case GGML_OP_POOL_AVG: res = 0; break;+ case GGML_OP_POOL_MAX: res = -FLT_MAX; break;+ }++ for (int i = bh; i < eh; i += 1) {+ for (int j = bw; j < ew; j += 1) {+ #if __CUDA_ARCH__ >= 350+ Ti cur = __ldg(i_ptr + i * iw + j);+ #else+ Ti cur = i_ptr[i * iw + j];+ #endif+ switch (op) {+ case GGML_OP_POOL_AVG: res += cur * scale; break;+ case GGML_OP_POOL_MAX: res = max(res, (To)cur); break;+ }+ }+ }+ o_ptr[cur_oh * ow + cur_ow] = res;+}+template<int qk, int qr, dequantize_kernel_t dq>static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {@@ -6393,6 +6848,16 @@ static void relu_f32_cuda(const float * x, float * dst, const int k, cudaStream_relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);}+static void hardsigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_HARDSIGMOID_BLOCK_SIZE - 1) / CUDA_HARDSIGMOID_BLOCK_SIZE;+ hardsigmoid_f32<<<num_blocks, CUDA_HARDSIGMOID_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}++static void hardswish_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_HARDSWISH_BLOCK_SIZE - 1) / CUDA_HARDSWISH_BLOCK_SIZE;+ hardswish_f32<<<num_blocks, CUDA_HARDSWISH_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}+static void leaky_relu_f32_cuda(const float * x, float * dst, const int k, const float negative_slope, cudaStream_t stream) {const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;leaky_relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k, negative_slope);@@ -6553,6 +7018,24 @@ static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k,dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);}+template<typename dst_t>+static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = (k + QK_K - 1) / QK_K;+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);+}+template <typename src_t, typename dst_t>static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;@@ -6590,6 +7073,12 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {return dequantize_row_iq2_xxs_cuda;case GGML_TYPE_IQ2_XS:return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;case GGML_TYPE_F32:return convert_unary_cuda<float>;default:@@ -6623,6 +7112,12 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {return dequantize_row_iq2_xxs_cuda;case GGML_TYPE_IQ2_XS:return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;case GGML_TYPE_F16:return convert_unary_cuda<half>;default:@@ -6727,112 +7222,85 @@ static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, floa<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-static void mul_mat_vec_q4_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK4_0 == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}--static void mul_mat_vec_q4_1_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK4_1 == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}--static void mul_mat_vec_q5_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK5_0 == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}--static void mul_mat_vec_q5_1_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK5_1 == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}--static void mul_mat_vec_q8_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK8_0 == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}--static void mul_mat_vec_q2_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}--static void mul_mat_vec_q3_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}+template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot>+static void mul_mat_vec_q_cuda(+ const void * vx, const void * vy, float * dst,+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {-static void mul_mat_vec_q4_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}+ GGML_ASSERT(ncols_x % qk == 0);+ GGML_ASSERT(ncols_y <= MMVQ_MAX_BATCH_SIZE);-static void mul_mat_vec_q5_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}+ int id;+ CUDA_CHECK(cudaGetDevice(&id));-static void mul_mat_vec_q6_K_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}+ int64_t nwarps = 1;+ int64_t rows_per_cuda_block = 1;-static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);-}+ if (g_device_caps[id].cc < CC_RDNA2) { // NVIDIA and AMD older than RDNA2+ switch(ncols_y) {+ case 1:+ nwarps = 4;+ rows_per_cuda_block = 1;+ break;+ case 2:+ case 3:+ case 4:+ nwarps = 4;+ rows_per_cuda_block = 2;+ break;+ case 5:+ case 6:+ case 7:+ case 8:+ nwarps = 2;+ rows_per_cuda_block = 2;+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ }+ const int64_t nblocks = (nrows_x + rows_per_cuda_block - 1) / rows_per_cuda_block;+ const dim3 block_nums(nblocks, 1, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);-static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);+ switch (ncols_y) {+ case 1:+ mul_mat_vec_q<1, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ case 2:+ mul_mat_vec_q<2, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ case 3:+ mul_mat_vec_q<3, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ case 4:+ mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ case 5:+ mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ case 6:+ mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ case 7:+ mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ case 8:+ mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);+ break;+ default:+ GGML_ASSERT(false);+ break;+ }}static void ggml_mul_mat_q4_0_q8_1_cuda(@@ -7307,69 +7775,82 @@ static void ggml_mul_mat_vec_nc_f16_f32_cuda((vx, y, dst, ncols_x, nrows_x, row_stride_x, channel_stride_x, nchannels_y/nchannels_x);}++static void ggml_cpy_f16_f32_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f16_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}+static void ggml_cpy_f32_f32_cuda(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;cpy_f32_f16<cpy_1_f32_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}static void ggml_cpy_f32_f16_cuda(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;cpy_f32_f16<cpy_1_f32_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}static void ggml_cpy_f32_q8_0_cuda(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK8_0 == 0);const int num_blocks = ne / QK8_0;cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}static void ggml_cpy_f32_q4_0_cuda(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK4_0 == 0);const int num_blocks = ne / QK4_0;cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}static void ggml_cpy_f32_q4_1_cuda(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK4_1 == 0);const int num_blocks = ne / QK4_1;cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}static void ggml_cpy_f16_f16_cuda(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int nb00, const int nb01, const int nb02,- const int ne10, const int ne11, const int nb10, const int nb11, const int nb12, cudaStream_t stream) {+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;cpy_f32_f16<cpy_1_f16_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12);+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}++static void scale_f32_cuda(const float * x, float * dst, const float scale, const int k, cudaStream_t stream) {const int num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;scale_f32<<<num_blocks, CUDA_SCALE_BLOCK_SIZE, 0, stream>>>(x, dst, scale, k);@@ -7448,7 +7929,7 @@ static void alibi_f32_cuda(const float * x, float * dst, const int ncols, conststatic void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {const dim3 block_dims(WARP_SIZE, 1, 1);- const dim3 block_nums(1, nrows, 1);+ const dim3 block_nums(nrows, 1, 1);k_sum_rows_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);}@@ -7474,100 +7955,65 @@ static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncolsdiag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);}-static void soft_max_f16_cuda(const float * x, const float * y, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, cudaStream_t stream) {- int nth = WARP_SIZE;- while (nth < ncols_x/2 && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;- const dim3 block_dims(nth, 1, 1);- const dim3 block_nums(nrows_x, 1, 1);- const size_t shmem = (GGML_PAD(ncols_x, 2*WARP_SIZE) + WARP_SIZE)*sizeof(half);- static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted.");- if (shmem <= g_device_caps[g_main_device].smpb) {- switch (ncols_x) {- case 32:- soft_max_f16<true, 32, 32, true><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- case 64:- soft_max_f16<true, 64, 32, false><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- case 128:- soft_max_f16<true, 128, 64, false><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- case 256:- soft_max_f16<true, 256, 128, false><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- case 512:- soft_max_f16<true, 512, 256, false><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- case 1024:- soft_max_f16<true, 1024, 512, false><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- case 2048:- soft_max_f16<true, 2048, 1024, false><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- case 4096:- soft_max_f16<true, 4096, 1024, false><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- default:- soft_max_f16<true, 0, 0, true><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- break;- }- } else {- const size_t shmem_low = WARP_SIZE*sizeof(half);- soft_max_f16<false, 0, 0, true><<<block_nums, block_dims, shmem_low, stream>>>(x, y, dst, ncols_x, nrows_y, scale);- }-}--static void soft_max_f32_cuda(const float * x, const float * y, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, cudaStream_t stream) {+static void soft_max_f32_cuda(const float * x, const float * mask, const float * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {int nth = WARP_SIZE;while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;const dim3 block_dims(nth, 1, 1);const dim3 block_nums(nrows_x, 1, 1);const size_t shmem = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE)*sizeof(float);static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted.");++ const uint32_t n_head_kv = nrows_x/nrows_y;+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv));++ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);+if (shmem < g_device_caps[g_main_device].smpb) {switch (ncols_x) {case 32:- soft_max_f32<true, 32, 32><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 32, 32><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 64:- soft_max_f32<true, 64, 64><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 64, 64><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 128:- soft_max_f32<true, 128, 128><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 128, 128><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 256:- soft_max_f32<true, 256, 256><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 256, 256><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 512:- soft_max_f32<true, 512, 512><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 512, 512><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 1024:- soft_max_f32<true, 1024, 1024><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 1024, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 2048:- soft_max_f32<true, 2048, 1024><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 2048, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 4096:- soft_max_f32<true, 4096, 1024><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 4096, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;default:- soft_max_f32<true, 0, 0><<<block_nums, block_dims, shmem, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<true, 0, 0><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;}} else {const size_t shmem_low = WARP_SIZE*sizeof(float);- soft_max_f32<false, 0, 0><<<block_nums, block_dims, shmem_low, stream>>>(x, y, dst, ncols_x, nrows_y, scale);+ soft_max_f32<false, 0, 0><<<block_nums, block_dims, shmem_low, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);}}-static void im2col_f32_f16_cuda(const float* x, half* dst,+template <typename T>+static void im2col_cuda(const float* x, T* dst,int IW, int IH, int OW, int OH, int KW, int KH, int IC,- int offset_delta,+ int batch, int batch_offset, int offset_delta,int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {const int parallel_elements = OW * KW * KH;const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;- dim3 block_nums(num_blocks, OH, IC);- im2col_f32_f16<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, offset_delta, IW, IH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);+ dim3 block_nums(num_blocks, OH, batch * IC);+ im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);}// buffer pool for cuda@@ -7836,6 +8282,8 @@ GGML_CALL void ggml_init_cublas() {if (cudaGetDeviceCount(&g_device_count) != cudaSuccess) {initialized = true;g_cublas_loaded = false;+ if (!FLAG_log_disable)+ fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__);return;}@@ -8166,6 +8614,34 @@ static void ggml_cuda_op_relu((void) src1_dd;}+static void ggml_cuda_op_hardsigmoid(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ hardsigmoid_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);++ (void) src1;+ (void) dst;+ (void) src1_dd;+}++static void ggml_cuda_op_hardswish(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ hardswish_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);++ (void) src1;+ (void) dst;+ (void) src1_dd;+}+static void ggml_cuda_op_leaky_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {@@ -8321,7 +8797,7 @@ static void ggml_cuda_op_mul_mat_q(CUDA_CHECK(cudaGetDevice(&id));// the main device has a larger memory buffer to hold the results from all GPUs- // nrows_dst == nrows of the matrix that the dequantize_mul_mat kernel writes into+ // nrows_dst == nrows of the matrix that the kernel writes intoconst int64_t nrows_dst = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff;switch (src0->type) {@@ -8399,6 +8875,9 @@ static int64_t get_row_rounding(ggml_type type, const std::array<float, GGML_CUDcase GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:return max_compute_capability >= CC_RDNA2 ? 128 : 64;default:GGML_ASSERT(false);@@ -8421,6 +8900,9 @@ static int64_t get_row_rounding(ggml_type type, const std::array<float, GGML_CUDcase GGML_TYPE_Q5_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:return max_compute_capability >= CC_VOLTA ? 128 : 64;case GGML_TYPE_Q6_K:return 64;@@ -8450,47 +8932,81 @@ static void ggml_cuda_op_mul_mat_vec_q(const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,const int64_t src1_padded_row_size, cudaStream_t stream) {- GGML_ASSERT(ggml_nrows(src1) == 1);-const int64_t ne00 = src0->ne[0];const int64_t row_diff = row_high - row_low;+ const int64_t ne10 = src1->ne[0];+ GGML_ASSERT(ne10 % QK8_1 == 0);++ const int64_t ne0 = dst->ne[0];++ int id;+ CUDA_CHECK(cudaGetDevice(&id));++ // the main device has a larger memory buffer to hold the results from all GPUs+ // nrows_dst == nrows of the matrix that the kernel writes into+ const int64_t nrows_dst = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff;+switch (src0->type) {case GGML_TYPE_Q4_0:- mul_mat_vec_q4_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q4_1:- mul_mat_vec_q4_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK4_1, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q5_0:- mul_mat_vec_q5_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q5_1:- mul_mat_vec_q5_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q8_0:- mul_mat_vec_q8_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q2_K:- mul_mat_vec_q2_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q3_K:- mul_mat_vec_q3_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q4_K:- mul_mat_vec_q4_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q5_K:- mul_mat_vec_q5_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_Q6_K:- mul_mat_vec_q6_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_IQ2_XXS:- mul_mat_vec_iq2_xxs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;case GGML_TYPE_IQ2_XS:- mul_mat_vec_iq2_xs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ mul_mat_vec_q_cuda<QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ3_XXS:+ mul_mat_vec_q_cuda<QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ1_S:+ mul_mat_vec_q_cuda<QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ4_NL:+ mul_mat_vec_q_cuda<QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);break;default:GGML_ASSERT(false);@@ -8526,9 +9042,9 @@ static void ggml_cuda_op_dequantize_mul_mat_vec(if (src1_convert_f16) {src1_dfloat = src1_dfloat_a.alloc(ne00);- ggml_cpy_f32_f16_cuda((const char *) src1_ddf_i, (char *) src1_dfloat, ne00,- ne00, 1, sizeof(float), 0, 0,- ne00, 1, sizeof(half), 0, 0, stream);+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ GGML_ASSERT(to_fp16_cuda != nullptr);+ to_fp16_cuda(src1_ddf_i, src1_dfloat, ne00, stream);}#elseconst dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion@@ -8793,13 +9309,46 @@ static void ggml_cuda_op_alibi((void) src1_dd;}+static void ggml_cuda_op_pool2d(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ const int32_t * opts = (const int32_t *)dst->op_params;+ enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);+ const int k0 = opts[1];+ const int k1 = opts[2];+ const int s0 = opts[3];+ const int s1 = opts[4];+ const int p0 = opts[5];+ const int p1 = opts[6];++ const int64_t IH = src0->ne[1];+ const int64_t IW = src0->ne[0];++ const int64_t N = dst->ne[3];+ const int64_t OC = dst->ne[2];+ const int64_t OH = dst->ne[1];+ const int64_t OW = dst->ne[0];++ const int parallel_elements = N * OC * OH * OW;+ const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE;+ dim3 block_nums(num_blocks);+ pool2d_nchw_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, main_stream>>>(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_dd, dst_dd, op);++ (void) src1;+ (void) src1_dd;+}+static void ggml_cuda_op_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F16);+ GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);const int32_t s0 = ((const int32_t*)(dst->op_params))[0];const int32_t s1 = ((const int32_t*)(dst->op_params))[1];@@ -8821,8 +9370,14 @@ static void ggml_cuda_op_im2col(const int64_t OW = dst->ne[1];const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32+ const int64_t batch = src1->ne[3];+ const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32- im2col_f32_f16_cuda(src1_dd, (half*) dst_dd, IW, IH, OW, OH, KW, KH, IC, delta_offset, s0, s1, p0, p1, d0, d1, main_stream);+ if(dst->type == GGML_TYPE_F16) {+ im2col_cuda(src1_dd, (half*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream);+ } else {+ im2col_cuda(src1_dd, (float*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream);+ }(void) src0;(void) src0_dd;@@ -8893,30 +9448,36 @@ static void ggml_cuda_op_soft_max(GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional- const int64_t ne00 = src0->ne[0];+ const int64_t ne00 = src0->ne[0];const int64_t nrows_x = ggml_nrows(src0);- const int64_t nrows_y = src1 ? ggml_nrows(src1) : 1;+ const int64_t nrows_y = src0->ne[1];- float scale = 1.0f;- memcpy(&scale, dst->op_params, sizeof(float));+ float scale = 1.0f;+ float max_bias = 0.0f;-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION >= CUDART_HMAX-#ifdef GGML_CUDA_F16- const bool use_f16_soft_max = true;-#else- const bool use_f16_soft_max = false;-#endif // GGML_CUDA_F16-#else- const bool use_f16_soft_max = false;-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && CUDART_VERSION >= CUDART_HMAX+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));+ memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));- if (use_f16_soft_max) {- soft_max_f16_cuda(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream);- } else {- soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream);+ // positions tensor+ float * src2_dd = nullptr;+ cuda_pool_alloc<float> src2_f;++ ggml_tensor * src2 = dst->src[2];+ const bool use_src2 = src2 != nullptr;++ if (use_src2) {+ const bool src2_on_device = src2->backend == GGML_BACKEND_GPU;++ if (src2_on_device) {+ ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra;+ src2_dd = (float *) src2_extra->data_device[g_main_device];+ } else {+ src2_dd = src2_f.alloc(ggml_nelements(src2));+ CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src2_dd, src2, 0, 0, 0, 1, main_stream));+ }}- (void) dst;+ soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, src2_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream);}static void ggml_cuda_op_scale(@@ -9051,9 +9612,15 @@ static void ggml_cuda_set_peer_access(const int n_tokens) {CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, id_other));if (can_access_peer) {if (enable_peer_access) {- CUDA_CHECK(cudaDeviceEnablePeerAccess(id_other, 0));+ cudaError_t err = cudaDeviceEnablePeerAccess(id_other, 0);+ if (err != cudaErrorPeerAccessAlreadyEnabled) {+ CUDA_CHECK(err);+ }} else {- CUDA_CHECK(cudaDeviceDisablePeerAccess(id_other));+ cudaError_t err = cudaDeviceDisablePeerAccess(id_other);+ if (err != cudaErrorPeerAccessNotEnabled) {+ CUDA_CHECK(err);+ }}}}@@ -9418,6 +9985,13 @@ static void ggml_cuda_relu(const ggml_tensor * src0, const ggml_tensor * src1, gggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_relu);}+static void ggml_cuda_hardsigmoid(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardsigmoid);+}++static void ggml_cuda_hardswish(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardswish);+}static void ggml_cuda_leaky_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_leaky_relu);}@@ -9553,7 +10127,7 @@ static __global__ void k_compute_batched_ptrs(ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;}-static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {CUBLAS_ENTRY();GGML_ASSERT(!ggml_is_transposed(src0));GGML_ASSERT(!ggml_is_transposed(src1));@@ -9712,39 +10286,69 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1int64_t min_compute_capability = INT_MAX;+ bool any_pascal_with_slow_fp16 = false;if (split) {ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;auto & tensor_split = buft_ctx->tensor_split;for (int id = 0; id < g_device_count; ++id) {- if (min_compute_capability > g_device_caps[id].cc && tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) {+ // skip devices that are not going to do any work:+ if (tensor_split[id] >= (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) {+ continue;+ }++ if (min_compute_capability > g_device_caps[id].cc) {min_compute_capability = g_device_caps[id].cc;}+ if (g_device_caps[id].cc == 610) {+ any_pascal_with_slow_fp16 = true;+ }}} else {- min_compute_capability = g_device_caps[g_main_device].cc;+ min_compute_capability = g_device_caps[g_main_device].cc;+ any_pascal_with_slow_fp16 = g_device_caps[g_main_device].cc == 610;}+ // check data types and tensor shapes for custom matrix multiplication kernels:+ bool use_dequantize_mul_mat_vec = (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16)+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32+ && src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->ne[1] == 1;++ bool use_mul_mat_vec_q = ggml_is_quantized(src0->type)+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32+ && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;++ bool use_mul_mat_q = ggml_cuda_supports_mmq(src0->type)+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)const bool fp16_performance_good = min_compute_capability >= CC_RDNA1;- bool use_mul_mat_q = ggml_is_quantized(src0->type);+#ifdef CUDA_USE_TENSOR_CORESuse_mul_mat_q = use_mul_mat_q && min_compute_capability < CC_RDNA3;#endif // CUDA_USE_TENSOR_CORES#else- const bool fp16_performance_good = min_compute_capability >= CC_VOLTA;- bool use_mul_mat_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type);+ // fp16 performance is good on Volta or newer and on P100 (compute capability 6.0)+ const bool fp16_performance_good = min_compute_capability >= CC_PASCAL && !any_pascal_with_slow_fp16;++ // mmvq and mmq need the __dp4a instruction which on NVIDIA is only available for CC >= 6.1+ use_mul_mat_vec_q = use_mul_mat_vec_q && min_compute_capability >= MIN_CC_DP4A;+ use_mul_mat_q = use_mul_mat_q && min_compute_capability >= MIN_CC_DP4A;+#ifdef CUDA_USE_TENSOR_CORES// when tensor cores are available, use them for large batch size// ref: https://github.com/ggerganov/llama.cpp/pull/3776- use_mul_mat_q = use_mul_mat_q && !(fp16_performance_good && src1->ne[1] > MMQ_MAX_BATCH_SIZE);+ use_mul_mat_q = use_mul_mat_q && (!fp16_performance_good || src1->ne[1] <= MMQ_MAX_BATCH_SIZE);#endif // CUDA_USE_TENSOR_CORES#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- use_mul_mat_q = use_mul_mat_q && ggml_cuda_supports_mmq(src0->type);+ // if mmvq is available it's a better choice than dmmv:+#ifndef GGML_CUDA_FORCE_DMMV+ use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q;+#endif // GGML_CUDA_FORCE_DMMV// debug helpers//printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);@@ -9762,32 +10366,15 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1ggml_cuda_mul_mat_vec_nc(src0, src1, dst);} else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {// KQ + KQV multi-batch- ggml_cuda_mul_mat_mat_batched_cublas(src0, src1, dst);- } else if (src0->type == GGML_TYPE_F32) {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false);- } else if (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16) {- if (src1->ne[1] == 1 && src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->type == GGML_TYPE_F32) {-#ifdef GGML_CUDA_FORCE_DMMV- const bool use_mul_mat_vec_q = false;-#else- const bool use_mul_mat_vec_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type) && ggml_nrows(src1) == 1;-#endif // GGML_CUDA_FORCE_DMMV-- if (use_mul_mat_vec_q) {- // NOTE: this kernel does not support ggml_nrows(src1) > 1- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true);- } else {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false);- }- } else {- if (use_mul_mat_q) {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true);- } else {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false);- }- }+ ggml_cuda_mul_mat_batched_cublas(src0, src1, dst);+ } else if (use_dequantize_mul_mat_vec) {+ ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false);+ } else if (use_mul_mat_vec_q) {+ ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true);+ } else if (use_mul_mat_q) {+ ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true);} else {- GGML_ASSERT(false);+ ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false);}}@@ -10129,19 +10716,25 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggconst int64_t ne00 = src0->ne[0];const int64_t ne01 = src0->ne[1];- GGML_ASSERT(src0->ne[3] == 1);+ const int64_t ne02 = src0->ne[2];++ //GGML_ASSERT(src0->ne[3] == 1);const int64_t nb00 = src0->nb[0];const int64_t nb01 = src0->nb[1];const int64_t nb02 = src0->nb[2];+ const int64_t nb03 = src0->nb[3];const int64_t ne10 = src1->ne[0];const int64_t ne11 = src1->ne[1];- GGML_ASSERT(src1->ne[3] == 1);+ const int64_t ne12 = src1->ne[2];++ //GGML_ASSERT(src1->ne[3] == 1);const int64_t nb10 = src1->nb[0];const int64_t nb11 = src1->nb[1];const int64_t nb12 = src1->nb[2];+ const int64_t nb13 = src1->nb[3];ggml_cuda_set_device(g_main_device);cudaStream_t main_stream = g_cudaStreams[g_main_device][0];@@ -10153,17 +10746,19 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggchar * src1_ddc = (char *) src1_extra->data_device[g_main_device];if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {- ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {- ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {- ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {- ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {- ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);} else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {- ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, nb00, nb01, nb02, ne10, ne11, nb10, nb11, nb12, main_stream);+ ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {+ ggml_cpy_f16_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);} else {fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,ggml_type_name(src0->type), ggml_type_name(src1->type));@@ -10196,6 +10791,10 @@ static void ggml_cuda_alibi(const ggml_tensor * src0, const ggml_tensor * src1,ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_alibi);}+static void ggml_cuda_pool2d(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_pool2d);+}+static void ggml_cuda_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_im2col);}@@ -10297,6 +10896,12 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, stcase GGML_UNARY_OP_RELU:func = ggml_cuda_relu;break;+ case GGML_UNARY_OP_HARDSIGMOID:+ func = ggml_cuda_hardsigmoid;+ break;+ case GGML_UNARY_OP_HARDSWISH:+ func = ggml_cuda_hardswish;+ break;default:return false;}@@ -10371,6 +10976,9 @@ GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, stcase GGML_OP_IM2COL:func = ggml_cuda_im2col;break;+ case GGML_OP_POOL_2D:+ func = ggml_cuda_pool2d;+ break;case GGML_OP_SUM_ROWS:func = ggml_cuda_sum_rows;break;@@ -10628,6 +11236,7 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {/* .get_name = */ ggml_backend_cuda_buffer_type_name,/* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX/* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,/* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend,/* .is_host = */ NULL,@@ -10684,10 +11293,10 @@ GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backeUNUSED(buffer);}-// unused at the moment-//static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {-// return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name;-//}+static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {+ return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name;+ UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds+}GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;@@ -10903,6 +11512,7 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface/* .get_name = */ ggml_backend_cuda_split_buffer_type_name,/* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX/* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,/* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend,/* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,@@ -10982,6 +11592,7 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {/* .get_name = */ ggml_backend_cuda_host_buffer_type_name,/* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX/* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,/* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend,/* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,@@ -11073,7 +11684,7 @@ GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggfor (int j = 0; j < GGML_MAX_SRC; j++) {if (node->src[j] != nullptr) {assert(node->src[j]->backend == GGML_BACKEND_GPU || node->src[j]->backend == GGML_BACKEND_GPU_SPLIT);- assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));+ assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer));assert(node->src[j]->extra != nullptr);}}@@ -11096,6 +11707,8 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, conscase GGML_UNARY_OP_GELU:case GGML_UNARY_OP_SILU:case GGML_UNARY_OP_RELU:+ case GGML_UNARY_OP_HARDSIGMOID:+ case GGML_UNARY_OP_HARDSWISH:case GGML_UNARY_OP_GELU_QUICK:case GGML_UNARY_OP_TANH:return true;@@ -11119,7 +11732,8 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, consreturn false;}ggml_type a_type = a->type;- if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS) {+ if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS ||+ a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL) {if (b->ne[1] == 1 && ggml_nrows(b) > 1) {return false;}@@ -11163,6 +11777,9 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, consif (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {return true;}+ if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {+ return true;+ }return false;} break;case GGML_OP_DUP:@@ -11191,6 +11808,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, conscase GGML_OP_ROPE:case GGML_OP_ALIBI:case GGML_OP_IM2COL:+ case GGML_OP_POOL_2D:case GGML_OP_SUM_ROWS:case GGML_OP_ARGSORT:case GGML_OP_ACC:@@ -11273,7 +11891,7 @@ GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params,extern "C" GGML_API GGML_CALL int ggml_backend_cuda_reg_devices();-GGML_CALL int ggml_backend_cuda_reg_devices() {+GGML_API GGML_CALL int ggml_backend_cuda_reg_devices() {int device_count = ggml_cuda_get_device_count();//int device_count = 1; // DEBUG: some tools require delaying CUDA initializationfor (int i = 0; i < device_count; i++) {diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex 50000b62..777fd29b 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -19,6 +19,7 @@ extern "C" {// fall back to the _Static_assert C11 keyword.// if C99 - static_assert is noop// ref: https://stackoverflow.com/a/53923785/4039976+#ifndef __cplusplus#ifndef static_assert#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)#define static_assert(cond, msg) _Static_assert(cond, msg)@@ -26,6 +27,7 @@ extern "C" {#define static_assert(cond, msg) struct global_scope_noop_trick#endif#endif+#endif// __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512#if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))@@ -49,9 +51,7 @@ extern "C" {//// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src///-#ifndef __CUDACC__#include <arm_neon.h>-#endif#define GGML_COMPUTE_FP16_TO_FP32(x) ((float) (x))#define GGML_COMPUTE_FP32_TO_FP16(x) (x)diff --git a/llama.cpp/ggml-metal.h b/llama.cpp/ggml-metal.hindex 7da759ff..1d536289 100644--- a/llama.cpp/ggml-metal.h+++ b/llama.cpp/ggml-metal.h@@ -59,6 +59,9 @@ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(voi// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdfGGML_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);+// capture all command buffers committed the next time `ggml_backend_graph_compute` is called+GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);+#ifdef __cplusplus}#endifdiff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex 95d2a9c8..25ca5ed2 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -49,10 +49,7 @@ void ggml_metal_link(const struct ggml_backend_api *backend_api) {#define UNUSED(x) (void)(x)-#define GGML_METAL_MAX_KERNELS 256-struct ggml_metal_kernel {- id<MTLFunction> function;id<MTLComputePipelineState> pipeline;};@@ -88,6 +85,9 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K,GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS,GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL,GGML_METAL_KERNEL_TYPE_GET_ROWS_I32,GGML_METAL_KERNEL_TYPE_RMS_NORM,GGML_METAL_KERNEL_TYPE_GROUP_NORM,@@ -109,6 +109,9 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32,//GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16,GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32,@@ -126,6 +129,9 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32,GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32,@@ -140,6 +146,9 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32,@@ -154,10 +163,14 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32,GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32,GGML_METAL_KERNEL_TYPE_ROPE_F32,GGML_METAL_KERNEL_TYPE_ROPE_F16,GGML_METAL_KERNEL_TYPE_ALIBI_F32,GGML_METAL_KERNEL_TYPE_IM2COL_F16,+ GGML_METAL_KERNEL_TYPE_IM2COL_F32,GGML_METAL_KERNEL_TYPE_UPSCALE_F32,GGML_METAL_KERNEL_TYPE_PAD_F32,GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,@@ -184,20 +197,21 @@ struct ggml_metal_context {id<MTLDevice> device;id<MTLCommandQueue> queue;- id<MTLLibrary> library;dispatch_queue_t d_queue;- struct ggml_metal_kernel kernels[GGML_METAL_MAX_KERNELS];+ struct ggml_metal_kernel kernels[GGML_METAL_KERNEL_TYPE_COUNT];bool support_simdgroup_reduction;bool support_simdgroup_mm;++ bool should_capture_next_compute;};// MSL code// TODO: move the contents here when ready// for now it is easier to work in a separate file-//static NSString * const msl_library_source = @"see metal.metal";+// static NSString * const msl_library_source = @"see metal.metal";// Here to assist with NSBundle Path Hack@interface GGMLMetalClass : NSObject@@ -273,6 +287,8 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {ctx->queue = [ctx->device newCommandQueue];ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);+ id<MTLLibrary> metal_library;+// load library{NSBundle * bundle = nil;@@ -287,12 +303,20 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {// pre-compiled library foundNSURL * libURL = [NSURL fileURLWithPath:libPath];GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]);- ctx->library = [ctx->device newLibraryWithURL:libURL error:&error];+ metal_library = [ctx->device newLibraryWithURL:libURL error:&error];if (error) {GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);return NULL;}} else {+#if GGML_METAL_EMBED_LIBRARY+ GGML_METAL_LOG_INFO("%s: using embedded metal library\n", __func__);++ extern const char ggml_metallib_start[];+ extern const char ggml_metallib_end[];++ NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];+#elseGGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);NSString * sourcePath;@@ -315,6 +339,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);return NULL;}+#endif@autoreleasepool {// dictionary of preprocessor macros@@ -329,7 +354,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {//[options setFastMathEnabled:false];- ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error];+ metal_library = [ctx->device newLibraryWithSource:src options:options error:&error];if (error) {GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);return NULL;@@ -378,6 +403,8 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false");GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");+ ctx->should_capture_next_compute = false;+#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)if (@available(macOS 10.12, iOS 16.0, *)) {GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);@@ -394,8 +421,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {{NSError * error = nil;- for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) {- ctx->kernels[i].function = nil;+ for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {ctx->kernels[i].pipeline = nil;}@@ -407,10 +433,12 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {#define GGML_METAL_ADD_KERNEL(e, name, supported) \if (supported) { \struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \- kernel->function = [ctx->library newFunctionWithName:@"kernel_"#name]; \- kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:kernel->function error:&error]; \+ id<MTLFunction> metal_function = [metal_library newFunctionWithName:@"kernel_"#name]; \+ kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:metal_function error:&error]; \+ [metal_function release]; \if (error) { \GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \+ [metal_library release]; \return NULL; \} \} else { \@@ -450,6 +478,9 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);@@ -471,6 +502,9 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);//GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);@@ -488,6 +522,9 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);@@ -502,6 +539,9 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);@@ -516,10 +556,14 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);@@ -539,23 +583,17 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);}+ [metal_library release];return ctx;}static void ggml_metal_free(struct ggml_metal_context * ctx) {GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);- for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) {- if (ctx->kernels[i].pipeline) {- [ctx->kernels[i].pipeline release];- }-- if (ctx->kernels[i].function) {- [ctx->kernels[i].function release];- }+ for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {+ [ctx->kernels[i].pipeline release];}- [ctx->library release];[ctx->queue release];[ctx->device release];@@ -650,6 +688,10 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constcase GGML_OP_ALIBI:case GGML_OP_ROPE:case GGML_OP_IM2COL:+ return true;+ case GGML_OP_POOL_1D:+ case GGML_OP_POOL_2D:+ return false;case GGML_OP_UPSCALE:case GGML_OP_PAD:case GGML_OP_ARGSORT:@@ -701,6 +743,7 @@ static bool ggml_metal_graph_compute(struct ggml_metal_context * ctx,struct ggml_cgraph * gf) {+ @autoreleasepool {MTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor;edesc.dispatchType = MTLDispatchTypeSerial;@@ -711,6 +754,20 @@ static bool ggml_metal_graph_compute(const int n_cb = ctx->n_cb;const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;+ const bool should_capture = ctx->should_capture_next_compute;+ if (should_capture) {+ ctx->should_capture_next_compute = false;++ MTLCaptureDescriptor * descriptor = [MTLCaptureDescriptor new];+ descriptor.captureObject = ctx->queue;++ NSError * error = nil;+ if (![[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]) {+ GGML_METAL_LOG_ERROR("%s: error: unable to start capture '%s'\n", __func__, [[error localizedDescription] UTF8String]);+ GGML_ASSERT(!"capture failed");+ }+ }+id<MTLCommandBuffer> command_buffer_builder[n_cb];for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];@@ -719,6 +776,7 @@ static bool ggml_metal_graph_compute(// enqueue the command buffers in order to specify their execution order[command_buffer enqueue];}+const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;dispatch_apply(n_cb, ctx->d_queue, ^(size_t iter) {@@ -726,6 +784,7 @@ static bool ggml_metal_graph_compute(size_t offs_src0 = 0;size_t offs_src1 = 0;+ size_t offs_src2 = 0;size_t offs_dst = 0;id<MTLCommandBuffer> command_buffer = command_buffers[cb_idx];@@ -744,6 +803,7 @@ static bool ggml_metal_graph_compute(struct ggml_tensor * src0 = gf->nodes[i]->src[0];struct ggml_tensor * src1 = gf->nodes[i]->src[1];+ struct ggml_tensor * src2 = gf->nodes[i]->src[2];struct ggml_tensor * dst = gf->nodes[i];switch (dst->op) {@@ -765,9 +825,9 @@ static bool ggml_metal_graph_compute(GGML_ASSERT(!"unsupported op");}-#ifndef GGML_METAL_NDEBUG- [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];-#endif+ if (should_capture) {+ [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];+ }const int64_t ne00 = src0 ? src0->ne[0] : 0;const int64_t ne01 = src0 ? src0->ne[1] : 0;@@ -805,6 +865,7 @@ static bool ggml_metal_graph_compute(id<MTLBuffer> id_src0 = src0 ? ggml_metal_get_buffer(src0, &offs_src0) : nil;id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(src1, &offs_src1) : nil;+ id<MTLBuffer> id_src2 = src2 ? ggml_metal_get_buffer(src2, &offs_src2) : nil;id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(dst, &offs_dst) : nil;//GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));@@ -1186,7 +1247,16 @@ static bool ggml_metal_graph_compute(pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline;}- const float scale = ((float *) dst->op_params)[0];+ const float scale = ((float *) dst->op_params)[0];+ const float max_bias = ((float *) dst->op_params)[1];++ const int64_t nrows_x = ggml_nrows(src0);+ const int64_t nrows_y = src0->ne[1];+ const uint32_t n_head_kv = nrows_x/nrows_y;+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv));++ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);[encoder setComputePipelineState:pipeline];[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];@@ -1195,11 +1265,20 @@ static bool ggml_metal_graph_compute(} else {[encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];}- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];- [encoder setBytes:&scale length:sizeof(scale) atIndex:6];+ if (id_src2) {+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];+ } else {+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:2];+ }+ [encoder setBuffer:id_dst offset:offs_dst atIndex:3];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:4];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:5];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:6];+ [encoder setBytes:&scale length:sizeof(scale) atIndex:7];+ [encoder setBytes:&max_bias length:sizeof(max_bias) atIndex:8];+ [encoder setBytes:&m0 length:sizeof(m0) atIndex:9];+ [encoder setBytes:&m1 length:sizeof(m1) atIndex:10];+ [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:11];[encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];[encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];@@ -1294,6 +1373,9 @@ static bool ggml_metal_graph_compute(case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break;case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break;case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break;+ case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32].pipeline; break;+ case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break;+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break;default: GGML_ASSERT(false && "MUL MAT-MAT not implemented");}@@ -1422,6 +1504,24 @@ static bool ggml_metal_graph_compute(nth1 = 16;pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline;} break;+ case GGML_TYPE_IQ3_XXS:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32].pipeline;+ } break;+ case GGML_TYPE_IQ1_S:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32].pipeline;+ } break;+ case GGML_TYPE_IQ4_NL:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32].pipeline;+ } break;default:{GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);@@ -1456,7 +1556,7 @@ static bool ggml_metal_graph_compute(if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 ||src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 ||- src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) {+ src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_IQ1_S) { // || src0t == GGML_TYPE_Q4_K) {[encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {@@ -1464,6 +1564,16 @@ static bool ggml_metal_graph_compute([encoder setThreadgroupMemoryLength:mem_size atIndex:0];[encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}+ else if (src0t == GGML_TYPE_IQ3_XXS) {+ const int mem_size = 256*4+128;+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src0t == GGML_TYPE_IQ4_NL) {+ const int mem_size = 32*sizeof(float);+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }else if (src0t == GGML_TYPE_Q4_K) {[encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}@@ -1500,8 +1610,6 @@ static bool ggml_metal_graph_compute(// max size of the src1ids array in the kernel stackGGML_ASSERT(ne11 <= 512);- struct ggml_tensor * src2 = gf->nodes[i]->src[2];-const int64_t ne20 = src2 ? src2->ne[0] : 0;const int64_t ne21 = src2 ? src2->ne[1] : 0;const int64_t ne22 = src2 ? src2->ne[2] : 0;@@ -1558,6 +1666,9 @@ static bool ggml_metal_graph_compute(case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break;case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break;case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break;+ case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32].pipeline; break;+ case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break;+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32 ].pipeline; break;default: GGML_ASSERT(false && "MUL_MAT_ID not implemented");}@@ -1689,6 +1800,24 @@ static bool ggml_metal_graph_compute(nth1 = 16;pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline;} break;+ case GGML_TYPE_IQ3_XXS:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32].pipeline;+ } break;+ case GGML_TYPE_IQ1_S:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32].pipeline;+ } break;+ case GGML_TYPE_IQ4_NL:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32].pipeline;+ } break;default:{GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);@@ -1739,7 +1868,7 @@ static bool ggml_metal_graph_compute(if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 ||src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 ||- src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) {+ src2t == GGML_TYPE_Q2_K || src2t == GGML_TYPE_IQ1_S) { // || src2t == GGML_TYPE_Q4_K) {[encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) {@@ -1747,6 +1876,16 @@ static bool ggml_metal_graph_compute([encoder setThreadgroupMemoryLength:mem_size atIndex:0];[encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}+ else if (src2t == GGML_TYPE_IQ3_XXS) {+ const int mem_size = 256*4+128;+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_IQ4_NL) {+ const int mem_size = 32*sizeof(float);+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }else if (src2t == GGML_TYPE_Q4_K) {[encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}@@ -1787,6 +1926,9 @@ static bool ggml_metal_graph_compute(case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break;case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break;case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break;+ case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS].pipeline; break;+ case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break;+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL ].pipeline; break;case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break;default: GGML_ASSERT(false && "not implemented");}@@ -1995,7 +2137,7 @@ static bool ggml_metal_graph_compute({GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F16);+ GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);const int32_t s0 = ((const int32_t *)(dst->op_params))[0];const int32_t s1 = ((const int32_t *)(dst->op_params))[1];@@ -2003,6 +2145,7 @@ static bool ggml_metal_graph_compute(const int32_t p1 = ((const int32_t *)(dst->op_params))[3];const int32_t d0 = ((const int32_t *)(dst->op_params))[4];const int32_t d1 = ((const int32_t *)(dst->op_params))[5];+const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;const int32_t N = src1->ne[is_2D ? 3 : 2];@@ -2023,8 +2166,8 @@ static bool ggml_metal_graph_compute(id<MTLComputePipelineState> pipeline = nil;- switch (src0->type) {- case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break;+ switch (dst->type) {+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F32].pipeline; break;case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break;default: GGML_ASSERT(false);};@@ -2217,9 +2360,9 @@ static bool ggml_metal_graph_compute(}}-#ifndef GGML_METAL_NDEBUG- [encoder popDebugGroup];-#endif+ if (should_capture) {+ [encoder popDebugGroup];+ }}[encoder endEncoding];@@ -2241,6 +2384,11 @@ static bool ggml_metal_graph_compute(}}+ if (should_capture) {+ [[MTLCaptureManager sharedCaptureManager] stopCapture];+ }++ }return true;}@@ -2409,6 +2557,16 @@ GGML_CALL static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backenUNUSED(buft);}+GGML_CALL static size_t ggml_backend_metal_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {+ id<MTLDevice> device = ggml_backend_metal_get_device();+ size_t max_size = device.maxBufferLength;+ ggml_backend_metal_free_device();++ return max_size;++ UNUSED(buft);+}+GGML_CALL static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {return ggml_backend_is_metal(backend) || ggml_backend_is_cpu(backend);@@ -2427,6 +2585,7 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {/* .get_name = */ ggml_backend_metal_buffer_type_get_name,/* .alloc_buffer = */ ggml_backend_metal_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,+ /* .get_max_size = */ ggml_backend_metal_buffer_type_get_max_size,/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes/* .supports_backend = */ ggml_backend_metal_buffer_type_supports_backend,/* .is_host = */ ggml_backend_metal_buffer_type_is_host,@@ -2560,7 +2719,6 @@ static struct ggml_backend_i ggml_backend_metal_i = {};void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {- if (log_callback) return; // [jart] unsafe across dso boundaryggml_metal_log_callback = log_callback;ggml_metal_log_user_data = user_data;}@@ -2602,6 +2760,13 @@ bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];}+void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {+ GGML_ASSERT(ggml_backend_is_metal(backend));++ struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;+ ctx->should_capture_next_compute = true;+}+GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warningGGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metalindex 029578dc..c223a981 100644--- a/llama.cpp/ggml-metal.metal+++ b/llama.cpp/ggml-metal.metal@@ -351,12 +351,17 @@ kernel void kernel_sum_rows(kernel void kernel_soft_max(device const float * src0,device const float * src1,+ device const float * src2,device float * dst,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,constant float & scale,- threadgroup float * buf [[threadgroup(0)]],+ constant float & max_bias,+ constant float & m0,+ constant float & m1,+ constant uint32_t & n_head_log2,+ threadgroup float * buf [[threadgroup(0)]],uint tgpig[[threadgroup_position_in_grid]],uint tpitg[[thread_position_in_threadgroup]],uint sgitg[[simdgroup_index_in_threadgroup]],@@ -368,13 +373,26 @@ kernel void kernel_soft_max(device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;device const float * pmask = src1 != src0 ? src1 + i01*ne00 : nullptr;+ device const float * ppos = src2 != src0 ? src2 : nullptr;device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;+ float slope = 0.0f;++ // ALiBi+ if (max_bias > 0.0f) {+ const int64_t h = i02;++ const float base = h < n_head_log2 ? m0 : m1;+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ slope = pow(base, exp);+ }+// parallel maxfloat lmax = -INFINITY;for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f));+ lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));}// find the max value in the block@@ -399,7 +417,7 @@ kernel void kernel_soft_max(// parallel sumfloat lsum = 0.0f;for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f)) - max_val);+ const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);lsum += exp_psrc0;pdst[i00] = exp_psrc0;}@@ -437,12 +455,17 @@ kernel void kernel_soft_max(kernel void kernel_soft_max_4(device const float * src0,device const float * src1,+ device const float * src2,device float * dst,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,constant float & scale,- threadgroup float * buf [[threadgroup(0)]],+ constant float & max_bias,+ constant float & m0,+ constant float & m1,+ constant uint32_t & n_head_log2,+ threadgroup float * buf [[threadgroup(0)]],uint tgpig[[threadgroup_position_in_grid]],uint tpitg[[thread_position_in_threadgroup]],uint sgitg[[simdgroup_index_in_threadgroup]],@@ -454,13 +477,25 @@ kernel void kernel_soft_max_4(device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);device const float4 * pmask = src1 != src0 ? (device const float4 *)(src1 + i01*ne00) : nullptr;+ device const float4 * ppos = src2 != src0 ? (device const float4 *)(src2) : nullptr;device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);+ float slope = 0.0f;++ if (max_bias > 0.0f) {+ const int64_t h = i02;++ const float base = h < n_head_log2 ? m0 : m1;+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ slope = pow(base, exp);+ }+// parallel maxfloat4 lmax4 = -INFINITY;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f));+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));}const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));@@ -486,7 +521,7 @@ kernel void kernel_soft_max_4(// parallel sumfloat4 lsum4 = 0.0f;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f)) - max_val);+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);lsum4 += exp_psrc4;pdst4[i00] = exp_psrc4;}@@ -1775,9 +1810,29 @@ kernel void kernel_rope(template [[host_name("kernel_rope_f32")]] kernel rope_t kernel_rope<float>;template [[host_name("kernel_rope_f16")]] kernel rope_t kernel_rope<half>;-kernel void kernel_im2col_f16(+typedef void (im2col_t)(+ device const float * x,+ device char * dst,+ constant int32_t & ofs0,+ constant int32_t & ofs1,+ constant int32_t & IW,+ constant int32_t & IH,+ constant int32_t & CHW,+ constant int32_t & s0,+ constant int32_t & s1,+ constant int32_t & p0,+ constant int32_t & p1,+ constant int32_t & d0,+ constant int32_t & d1,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tgpg[[threadgroups_per_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]);++template <typename T>+kernel void kernel_im2col(device const float * x,- device half * dst,+ device char * dst,constant int32_t & ofs0,constant int32_t & ofs1,constant int32_t & IW,@@ -1800,14 +1855,19 @@ kernel void kernel_im2col_f16((tpitg[0] * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * CHW +(tgpig[0] * (ntg[1] * ntg[2]) + tpitg[1] * ntg[2] + tpitg[2]);+ device T * pdst = (device T *) (dst);+if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {- dst[offset_dst] = 0.0f;+ pdst[offset_dst] = 0.0f;} else {const int32_t offset_src = tpitg[0] * ofs0 + tgpig[0] * ofs1;- dst[offset_dst] = x[offset_src + iih * IW + iiw];+ pdst[offset_dst] = x[offset_src + iih * IW + iiw];}}+template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col<float>;+template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col<half>;+kernel void kernel_upscale_f32(device const char * src0,device char * dst,@@ -2459,6 +2519,25 @@ typedef struct {} block_iq2_xs;// 74 bytes / block for QK_K = 256, so 2.3125 bpw+typedef struct {+ half d;+ uint8_t qs[3*QK_K/8];+} block_iq3_xxs;+// 98 bytes / block for QK_K = 256, so 3.0625 bpw++typedef struct {+ half d;+ uint8_t qs[QK_K/8];+ uint8_t scales[QK_K/16];+} block_iq1_s;++// Non-linear quants+#define QK4_NL 32+typedef struct {+ half d;+ uint8_t qs[QK4_NL/2];+} block_iq4_nl;+//====================================== dot products =========================void kernel_mul_mv_q2_K_f32_impl(@@ -3681,6 +3760,173 @@ constexpr constant static uint64_t iq2xs_grid[512] = {0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,};+constexpr constant static uint32_t iq3xxs_grid[256] = {+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,+};++#define NGRID_IQ1S 512+constexpr constant static uint64_t iq1s_grid[NGRID_IQ1S] = {+ 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,+ 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,+ 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,+ 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,+ 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,+ 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,+ 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,+ 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,+ 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,+ 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,+ 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,+ 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,+ 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,+ 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,+ 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,+ 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,+ 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,+ 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,+ 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,+ 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,+ 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,+ 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,+ 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,+ 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,+ 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,+ 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,+ 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,+ 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,+ 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,+ 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,+ 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,+ 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,+ 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,+ 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,+ 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,+ 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,+ 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,+ 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,+ 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,+ 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,+ 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,+ 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,+ 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,+ 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,+ 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,+ 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,+ 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,+ 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,+ 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,+ 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,+ 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,+ 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,+ 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,+ 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,+ 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,+ 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,+ 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,+ 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,+ 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,+ 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,+ 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,+ 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,+ 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,+ 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,+ 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,+ 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,+ 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,+ 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,+ 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,+ 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,+ 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,+ 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,+ 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,+ 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,+ 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,+ 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,+ 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,+ 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,+ 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,+ 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,+ 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,+ 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,+ 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,+ 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,+ 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,+ 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,+ 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,+ 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,+ 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,+ 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,+ 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,+ 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,+ 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,+ 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,+ 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,+ 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,+ 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,+ 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,+ 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,+ 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,+ 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,+ 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,+ 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,+ 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,+ 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,+ 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,+ 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,+ 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,+ 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,+ 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,+ 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,+ 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,+ 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,+ 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,+ 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,+ 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,+ 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,+ 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,+ 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,+ 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,+ 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,+ 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,+ 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,+ 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,+ 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,+ 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,+};+constexpr constant static uint8_t ksigns_iq2xs[128] = {0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,@@ -3787,7 +4033,10 @@ void kernel_mul_mv_iq2_xxs_f32_impl(y4 += 32 * 32;}#else- // TODO+ (void) x;+ (void) y;+ (void) yl;+ (void) nb32;#endiffor (int row = 0; row < N_DST; ++row) {@@ -3930,7 +4179,10 @@ void kernel_mul_mv_iq2_xs_f32_impl(y4 += 32 * 32;}#else- // TODO+ (void) x;+ (void) y;+ (void) yl;+ (void) nb32;#endiffor (int row = 0; row < N_DST; ++row) {@@ -3970,157 +4222,541 @@ kernel void kernel_mul_mv_iq2_xs_f32(kernel_mul_mv_iq2_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);}-//============================= templates and their specializations =============================--// NOTE: this is not dequantizing - we are simply fitting the template-template <typename type4x4>-void dequantize_f32(device const float4x4 * src, short il, thread type4x4 & reg) {- float4x4 temp = *(((device float4x4 *)src));- for (int i = 0; i < 16; i++){- reg[i/4][i%4] = temp[i/4][i%4];- }-}+void kernel_mul_mv_iq3_xxs_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup int8_t * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {-template <typename type4x4>-void dequantize_f16(device const half4x4 * src, short il, thread type4x4 & reg) {- half4x4 temp = *(((device half4x4 *)src));- for (int i = 0; i < 16; i++){- reg[i/4][i%4] = temp[i/4][i%4];- }-}+ const int nb = ne00/QK_K;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;-template <typename type4x4>-void dequantize_q4_0(device const block_q4_0 *xb, short il, thread type4x4 & reg) {- device const uint16_t * qs = ((device const uint16_t *)xb + 1);- const float d1 = il ? (xb->d / 16.h) : xb->d;- const float d2 = d1 / 256.f;- const float md = -8.h * xb->d;- const ushort mask0 = il ? 0x00F0 : 0x000F;- const ushort mask1 = mask0 << 8;+ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;+ const int ib_row = first_row * nb;- for (int i=0;i<8;i++) {- reg[i/2][2*(i%2)+0] = d1 * (qs[i] & mask0) + md;- reg[i/2][2*(i%2)+1] = d2 * (qs[i] & mask1) + md;- }-}+ const uint i12 = im%ne12;+ const uint i13 = im/ne12;-template <typename type4x4>-void dequantize_q4_1(device const block_q4_1 *xb, short il, thread type4x4 & reg) {- device const uint16_t * qs = ((device const uint16_t *)xb + 2);- const float d1 = il ? (xb->d / 16.h) : xb->d;- const float d2 = d1 / 256.f;- const float m = xb->m;- const ushort mask0 = il ? 0x00F0 : 0x000F;- const ushort mask1 = mask0 << 8;+ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);- for (int i=0;i<8;i++) {- reg[i/2][2*(i%2)+0] = ((qs[i] & mask0) * d1) + m;- reg[i/2][2*(i%2)+1] = ((qs[i] & mask1) * d2) + m;- }-}+ device const block_iq3_xxs * x = (device const block_iq3_xxs *) src0 + ib_row + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-template <typename type4x4>-void dequantize_q5_0(device const block_q5_0 *xb, short il, thread type4x4 & reg) {- device const uint16_t * qs = ((device const uint16_t *)xb + 3);- const float d = xb->d;- const float md = -16.h * xb->d;- const ushort mask = il ? 0x00F0 : 0x000F;+ float yl[32];+ float sumf[N_DST]={0.f}, all_sum;- const uint32_t qh = *((device const uint32_t *)xb->qh);+ const int nb32 = nb * (QK_K / 32);- const int x_mv = il ? 4 : 0;+ threadgroup uint32_t * values = (threadgroup uint32_t *)shared_values;+ threadgroup uint8_t * shared_signs = (threadgroup uint8_t *)(values + 256);+ {+ int nval = 4;+ int pos = (32*sgitg + tiisg)*nval;+ for (int i = 0; i < nval; ++i) values[pos + i] = iq3xxs_grid[pos + i];+ nval = 2;+ pos = (32*sgitg + tiisg)*nval;+ for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }- const int gh_mv = il ? 12 : 0;- const int gh_bk = il ? 0 : 4;+#if QK_K == 256+ const int ix = tiisg;- for (int i = 0; i < 8; i++) {- // extract the 5-th bits for x0 and x1- const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;- const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;+ device const float * y4 = y + 32 * ix;- // combine the 4-bits from qs with the 5th bit- const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);- const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {- reg[i/2][2*(i%2)+0] = d * x0 + md;- reg[i/2][2*(i%2)+1] = d * x1 + md;- }-}+ for (int i = 0; i < 32; ++i) {+ yl[i] = y4[i];+ }-template <typename type4x4>-void dequantize_q5_1(device const block_q5_1 *xb, short il, thread type4x4 & reg) {- device const uint16_t * qs = ((device const uint16_t *)xb + 4);- const float d = xb->d;- const float m = xb->m;- const ushort mask = il ? 0x00F0 : 0x000F;+ const int ibl = ib32 / (QK_K / 32);+ const int ib = ib32 % (QK_K / 32);- const uint32_t qh = *((device const uint32_t *)xb->qh);+ device const block_iq3_xxs * xr = x + ibl;+ device const uint8_t * q3 = xr->qs + 8 * ib;+ device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib;+ device const half * dh = &xr->d;- const int x_mv = il ? 4 : 0;+ for (int row = 0; row < N_DST; row++) {- const int gh_mv = il ? 12 : 0;- const int gh_bk = il ? 0 : 4;+ const float db = dh[0];+ const uint32_t aux32 = gas[0] | (gas[1] << 16);+ const float d = db * (0.5f + (aux32 >> 28));- for (int i = 0; i < 8; i++) {- // extract the 5-th bits for x0 and x1- const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;- const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;+ float2 sum = {0};+ for (int l = 0; l < 4; ++l) {+ const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(values + q3[2*l+0]);+ const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(values + q3[2*l+1]);+ const uint8_t signs = shared_signs[(aux32 >> 7*l) & 127];+ for (int j = 0; j < 4; ++j) {+ sum[0] += yl[8*l + j + 0] * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ sum[1] += yl[8*l + j + 4] * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+ }+ sumf[row] += d * (sum[0] + sum[1]);- // combine the 4-bits from qs with the 5th bit- const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);- const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);+ dh += nb*sizeof(block_iq3_xxs)/2;+ q3 += nb*sizeof(block_iq3_xxs);+ gas += nb*sizeof(block_iq3_xxs)/2;+ }- reg[i/2][2*(i%2)+0] = d * x0 + m;- reg[i/2][2*(i%2)+1] = d * x1 + m;+ y4 += 32 * 32;}-}--template <typename type4x4>-void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) {- device const int8_t * qs = ((device const int8_t *)xb->qs);- const half d = xb->d;+#else+ (void) x;+ (void) y;+ (void) yl;+ (void) nb32;+#endif- for (int i = 0; i < 16; i++) {- reg[i/4][i%4] = (qs[i + 16*il] * d);+ for (int row = 0; row < N_DST; ++row) {+ all_sum = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.5f;+ }}}-template <typename type4x4>-void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg) {- const float d = xb->d;- const float min = xb->dmin;- device const uint8_t * q = (device const uint8_t *)xb->qs;- float dl, ml;- uint8_t sc = xb->scales[il];+[[host_name("kernel_mul_mv_iq3_xxs_f32")]]+kernel void kernel_mul_mv_iq3_xxs_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup int8_t * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {-#if QK_K == 256- q = q + 32*(il/8) + 16*(il&1);- il = (il/2)%4;-#endif- half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h);- uchar mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);- dl = d * (sc & 0xF) * coef, ml = min * (sc >> 4);- for (int i = 0; i < 16; ++i) {- reg[i/4][i%4] = dl * (q[i] & mask) - ml;- }+ kernel_mul_mv_iq3_xxs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);}-template <typename type4x4>-void dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg) {- const half d_all = xb->d;- device const uint8_t * q = (device const uint8_t *)xb->qs;- device const uint8_t * h = (device const uint8_t *)xb->hmask;- device const int8_t * scales = (device const int8_t *)xb->scales;--#if QK_K == 256- q = q + 32 * (il/8) + 16 * (il&1);- h = h + 16 * (il&1);- uint8_t m = 1 << (il/2);- uint16_t kmask1 = (il/4)>1 ? ((il/4)>2 ? 192 : 48) : \- ((il/4)>0 ? 12 : 3);- uint16_t kmask2 = il/8 ? 0xF0 : 0x0F;- uint16_t scale_2 = scales[il%8], scale_1 = scales[8 + il%4];- int16_t dl_int = (il/4)&1 ? (scale_2&kmask2) | ((scale_1&kmask1) << 2)+void kernel_mul_mv_iq1_s_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int nb = ne00/QK_K;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;+ const int ib_row = first_row * nb;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+ device const block_iq1_s * x = (device const block_iq1_s *) src0 + ib_row + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[16];+ float sumf[N_DST]={0.f}, all_sum;++ const int nb32 = nb * (QK_K / 32);++#if QK_K == 256+ const int ix = tiisg/2;+ const int il = tiisg%2;++ device const float * y4 = y + 32 * ix + 16 * il;++ for (int ib32 = ix; ib32 < nb32; ib32 += 16) {++ for (int i = 0; i < 16; ++i) {+ yl[i] = y4[i];+ }++ const int ibl = ib32 / (QK_K / 32);+ const int ib = ib32 % (QK_K / 32);++ device const block_iq1_s * xr = x + ibl;+ device const uint8_t * qs = xr->qs + 4 * ib + 2 * il;+ device const uint8_t * sc = xr->scales + 2 * ib + il;+ device const half * dh = &xr->d;++ for (int row = 0; row < N_DST; row++) {++ constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5)));+ constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1)));++ float2 sum = {0};+ for (int j = 0; j < 8; ++j) {+ sum[0] += yl[j+ 0] * grid1[j];+ sum[1] += yl[j+ 8] * grid2[j];+ }+ sumf[row] += (float)dh[0] * (sum[0] * (2*(sc[0] & 7) + 1) + sum[1] * (2*((sc[0] >> 4) & 7) + 1));++ dh += nb*sizeof(block_iq1_s)/2;+ qs += nb*sizeof(block_iq1_s);+ sc += nb*sizeof(block_iq1_s);+ }++ y4 += 16 * 32;+ }+#else+ (void) x;+ (void) y;+ (void) yl;+ (void) nb32;+#endif++ for (int row = 0; row < N_DST; ++row) {+ all_sum = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;+ }+ }+}++constexpr constant static float kvalues_iq4nl_f[16] = {+ -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f+};++void kernel_mul_mv_iq4_nl_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup float * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int nb = ne00/QK4_NL;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;+ const int first_row = (r0 * 2 + sgitg) * 2;+ const int ib_row = first_row * nb;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);+ device const block_iq4_nl * x = (device const block_iq4_nl *) src0 + ib_row + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ const int ix = tiisg/2; // 0...15+ const int it = tiisg%2; // 0 or 1++ shared_values[tiisg] = kvalues_iq4nl_f[tiisg%16];+ threadgroup_barrier(mem_flags::mem_threadgroup);++ float4 yl[4];+ float sumf[2]={0.f}, all_sum;++ device const float * yb = y + ix * QK4_NL + it * 8;++ uint32_t aux32[2];+ thread const uint8_t * q8 = (thread const uint8_t *)aux32;++ float4 qf1, qf2;++ for (int ib = ix; ib < nb; ib += 16) {++ device const float4 * y4 = (device const float4 *)yb;+ yl[0] = y4[0]; yl[1] = y4[4]; yl[2] = y4[1]; yl[3] = y4[5];++ for (int row = 0; row < 2; ++row) {++ device const block_iq4_nl & xb = x[row*nb + ib];+ device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it);++ float4 acc1 = {0.f}, acc2 = {0.f};++ aux32[0] = q4[0] | (q4[1] << 16);+ aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;+ aux32[0] &= 0x0f0f0f0f;+ qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]};+ qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]};+ acc1 += yl[0] * qf1;+ acc2 += yl[1] * qf2;++ aux32[0] = q4[2] | (q4[3] << 16);+ aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;+ aux32[0] &= 0x0f0f0f0f;+ qf1 = {shared_values[q8[0]], shared_values[q8[1]], shared_values[q8[2]], shared_values[q8[3]]};+ qf2 = {shared_values[q8[4]], shared_values[q8[5]], shared_values[q8[6]], shared_values[q8[7]]};+ acc1 += yl[2] * qf1;+ acc2 += yl[3] * qf2;++ acc1 += acc2;++ sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]);++ }++ yb += 16 * QK4_NL;+ }++ for (int row = 0; row < 2; ++row) {+ all_sum = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;+ }+ }+}++[[host_name("kernel_mul_mv_iq1_s_f32")]]+kernel void kernel_mul_mv_iq1_s_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_iq1_s_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}++[[host_name("kernel_mul_mv_iq4_nl_f32")]]+kernel void kernel_mul_mv_iq4_nl_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup float * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_iq4_nl_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);+}++//============================= templates and their specializations =============================++// NOTE: this is not dequantizing - we are simply fitting the template+template <typename type4x4>+void dequantize_f32(device const float4x4 * src, short il, thread type4x4 & reg) {+ float4x4 temp = *(((device float4x4 *)src));+ for (int i = 0; i < 16; i++){+ reg[i/4][i%4] = temp[i/4][i%4];+ }+}++template <typename type4x4>+void dequantize_f16(device const half4x4 * src, short il, thread type4x4 & reg) {+ half4x4 temp = *(((device half4x4 *)src));+ for (int i = 0; i < 16; i++){+ reg[i/4][i%4] = temp[i/4][i%4];+ }+}++template <typename type4x4>+void dequantize_q4_0(device const block_q4_0 *xb, short il, thread type4x4 & reg) {+ device const uint16_t * qs = ((device const uint16_t *)xb + 1);+ const float d1 = il ? (xb->d / 16.h) : xb->d;+ const float d2 = d1 / 256.f;+ const float md = -8.h * xb->d;+ const ushort mask0 = il ? 0x00F0 : 0x000F;+ const ushort mask1 = mask0 << 8;++ for (int i=0;i<8;i++) {+ reg[i/2][2*(i%2)+0] = d1 * (qs[i] & mask0) + md;+ reg[i/2][2*(i%2)+1] = d2 * (qs[i] & mask1) + md;+ }+}++template <typename type4x4>+void dequantize_q4_1(device const block_q4_1 *xb, short il, thread type4x4 & reg) {+ device const uint16_t * qs = ((device const uint16_t *)xb + 2);+ const float d1 = il ? (xb->d / 16.h) : xb->d;+ const float d2 = d1 / 256.f;+ const float m = xb->m;+ const ushort mask0 = il ? 0x00F0 : 0x000F;+ const ushort mask1 = mask0 << 8;++ for (int i=0;i<8;i++) {+ reg[i/2][2*(i%2)+0] = ((qs[i] & mask0) * d1) + m;+ reg[i/2][2*(i%2)+1] = ((qs[i] & mask1) * d2) + m;+ }+}++template <typename type4x4>+void dequantize_q5_0(device const block_q5_0 *xb, short il, thread type4x4 & reg) {+ device const uint16_t * qs = ((device const uint16_t *)xb + 3);+ const float d = xb->d;+ const float md = -16.h * xb->d;+ const ushort mask = il ? 0x00F0 : 0x000F;++ const uint32_t qh = *((device const uint32_t *)xb->qh);++ const int x_mv = il ? 4 : 0;++ const int gh_mv = il ? 12 : 0;+ const int gh_bk = il ? 0 : 4;++ for (int i = 0; i < 8; i++) {+ // extract the 5-th bits for x0 and x1+ const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;+ const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;++ // combine the 4-bits from qs with the 5th bit+ const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);+ const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);++ reg[i/2][2*(i%2)+0] = d * x0 + md;+ reg[i/2][2*(i%2)+1] = d * x1 + md;+ }+}++template <typename type4x4>+void dequantize_q5_1(device const block_q5_1 *xb, short il, thread type4x4 & reg) {+ device const uint16_t * qs = ((device const uint16_t *)xb + 4);+ const float d = xb->d;+ const float m = xb->m;+ const ushort mask = il ? 0x00F0 : 0x000F;++ const uint32_t qh = *((device const uint32_t *)xb->qh);++ const int x_mv = il ? 4 : 0;++ const int gh_mv = il ? 12 : 0;+ const int gh_bk = il ? 0 : 4;++ for (int i = 0; i < 8; i++) {+ // extract the 5-th bits for x0 and x1+ const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10;+ const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10;++ // combine the 4-bits from qs with the 5th bit+ const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0);+ const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1);++ reg[i/2][2*(i%2)+0] = d * x0 + m;+ reg[i/2][2*(i%2)+1] = d * x1 + m;+ }+}++template <typename type4x4>+void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) {+ device const int8_t * qs = ((device const int8_t *)xb->qs);+ const half d = xb->d;++ for (int i = 0; i < 16; i++) {+ reg[i/4][i%4] = (qs[i + 16*il] * d);+ }+}++template <typename type4x4>+void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg) {+ const float d = xb->d;+ const float min = xb->dmin;+ device const uint8_t * q = (device const uint8_t *)xb->qs;+ float dl, ml;+ uint8_t sc = xb->scales[il];++#if QK_K == 256+ q = q + 32*(il/8) + 16*(il&1);+ il = (il/2)%4;+#endif+ half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h);+ uchar mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3);+ dl = d * (sc & 0xF) * coef, ml = min * (sc >> 4);+ for (int i = 0; i < 16; ++i) {+ reg[i/4][i%4] = dl * (q[i] & mask) - ml;+ }+}++template <typename type4x4>+void dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg) {+ const half d_all = xb->d;+ device const uint8_t * q = (device const uint8_t *)xb->qs;+ device const uint8_t * h = (device const uint8_t *)xb->hmask;+ device const int8_t * scales = (device const int8_t *)xb->scales;++#if QK_K == 256+ q = q + 32 * (il/8) + 16 * (il&1);+ h = h + 16 * (il&1);+ uint8_t m = 1 << (il/2);+ uint16_t kmask1 = (il/4)>1 ? ((il/4)>2 ? 192 : 48) : \+ ((il/4)>0 ? 12 : 3);+ uint16_t kmask2 = il/8 ? 0xF0 : 0x0F;+ uint16_t scale_2 = scales[il%8], scale_1 = scales[8 + il%4];+ int16_t dl_int = (il/4)&1 ? (scale_2&kmask2) | ((scale_1&kmask1) << 2): (scale_2&kmask2) | ((scale_1&kmask1) << 4);float dl = il<8 ? d_all * (dl_int - 32.f) : d_all * (dl_int / 16.f - 32.f);const float ml = 4.f * dl;@@ -4165,6 +4801,8 @@ void dequantize_q4_K(device const block_q4_K *xb, short il, thread type4x4 & regconst float dl = d * sc[0];const float ml = min * sc[1];#else+ (void) get_scale_min_k4_just2;+q = q + 16 * (il&1);device const uint8_t * s = xb->scales;device const half2 * dh = (device const half2 *)xb->d;@@ -4287,6 +4925,64 @@ void dequantize_iq2_xs(device const block_iq2_xs * xb, short il, thread type4x4}}+template <typename type4x4>+void dequantize_iq3_xxs(device const block_iq3_xxs * xb, short il, thread type4x4 & reg) {+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2+ const float d = xb->d;+ const int ib32 = il/2;+ il = il%2;+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16+ device const uint8_t * q3 = xb->qs + 8*ib32;+ device const uint16_t * gas = (device const uint16_t *)(xb->qs + QK_K/4) + 2*ib32;+ const uint32_t aux32 = gas[0] | (gas[1] << 16);+ const float dl = d * (0.5f + (aux32 >> 28)) * 0.5f;+ constant uint8_t * grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+0]);+ constant uint8_t * grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+1]);+ uint8_t signs = ksigns_iq2xs[(aux32 >> 14*il) & 127];+ for (int i = 0; i < 4; ++i) {+ reg[0][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f);+ reg[1][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f);+ }+ grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+2]);+ grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+3]);+ signs = ksigns_iq2xs[(aux32 >> (14*il+7)) & 127];+ for (int i = 0; i < 4; ++i) {+ reg[2][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f);+ reg[3][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f);+ }+}++template <typename type4x4>+void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) {+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2+ const float d = xb->d;+ device const uint8_t * qs = xb->qs + 2*il;+ device const uint8_t * sc = xb->scales + il;+ const float dl1 = d * (2*(sc[0] & 7) + 1);+ const float dl2 = d * (2*((sc[0] >> 4) & 7) + 1);+ constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5)));+ constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1)));+ for (int i = 0; i < 8; ++i) {+ reg[i/4+0][i%4] = dl1 * grid1[i];+ reg[i/4+2][i%4] = dl2 * grid2[i];+ }+}++template <typename type4x4>+void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 & reg) {+ device const uint16_t * q4 = (device const uint16_t *)xb->qs;+ const float d = xb->d;+ uint32_t aux32;+ thread const uint8_t * q8 = (thread const uint8_t *)&aux32;+ for (int i = 0; i < 4; ++i) {+ aux32 = ((q4[2*i] | (q4[2*i+1] << 16)) >> 4*il) & 0x0f0f0f0f;+ reg[i][0] = d * kvalues_iq4nl_f[q8[0]];+ reg[i][1] = d * kvalues_iq4nl_f[q8[1]];+ reg[i][2] = d * kvalues_iq4nl_f[q8[2]];+ reg[i][3] = d * kvalues_iq4nl_f[q8[3]];+ }+}+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>kernel void kernel_get_rows(device const void * src0,@@ -4828,6 +5524,9 @@ template [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_t kernel_get_rowstemplate [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_t kernel_get_rows<block_q6_K, QK_NL, dequantize_q6_K>;template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_t kernel_get_rows<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_t kernel_get_rows<block_iq2_xs, QK_NL, dequantize_iq2_xs>;+template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_t kernel_get_rows<block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;+template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows<block_iq1_s, QK_NL, dequantize_iq1_s>;+template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows<block_iq4_nl, 2, dequantize_iq4_nl>;//// matrix-matrix multiplication@@ -4866,6 +5565,9 @@ template [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mat_mm_t kernel_mul_mm<btemplate [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm<block_q6_K, QK_NL, dequantize_q6_K>;template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq2_xs, QK_NL, dequantize_iq2_xs>;+template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;+template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq1_s, QK_NL, dequantize_iq1_s>;+template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq4_nl, 2, dequantize_iq4_nl>;//// indirect matrix-matrix multiplication@@ -4916,6 +5618,9 @@ template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mat_mm_id_t kernel_mutemplate [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q6_K, QK_NL, dequantize_q6_K>;template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_xs, QK_NL, dequantize_iq2_xs>;+template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq3_xxs, QK_NL, dequantize_iq3_xxs>;+template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq1_s, QK_NL, dequantize_iq1_s>;+template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq4_nl, 2, dequantize_iq4_nl>;//// matrix-vector multiplication@@ -5818,3 +6523,196 @@ kernel void kernel_mul_mv_id_iq2_xs_f32(tiisg,sgitg);}++[[host_name("kernel_mul_mv_id_iq3_xxs_f32")]]+kernel void kernel_mul_mv_id_iq3_xxs_f32(+ device const char * ids,+ device const char * src1,+ device float * dst,+ constant uint64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ threadgroup int8_t * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_iq3_xxs_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ dst + bid*ne0,+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ shared_values,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_iq1_s_f32")]]+kernel void kernel_mul_mv_id_iq1_s_f32(+ device const char * ids,+ device const char * src1,+ device float * dst,+ constant uint64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_iq1_s_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ dst + bid*ne0,+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ tgpig,+ tiisg,+ sgitg);+}++[[host_name("kernel_mul_mv_id_iq4_nl_f32")]]+kernel void kernel_mul_mv_id_iq4_nl_f32(+ device const char * ids,+ device const char * src1,+ device float * dst,+ constant uint64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ threadgroup float * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_iq4_nl_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ dst + bid*ne0,+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ shared_values,+ tgpig,+ tiisg,+ sgitg);+}diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex af83528b..80f6e745 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -44,12 +44,16 @@#include <riscv_vector.h>#endif+#define UNUSED GGML_UNUSED+#undef MIN#undef MAX#define MIN(a, b) ((a) < (b) ? (a) : (b))#define MAX(a, b) ((a) > (b) ? (a) : (b))+#define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) }+#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)// horizontally add 8 floats@@ -3466,6 +3470,174 @@ static const uint64_t iq2xs_grid[512] = {0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,};+static const uint32_t iq3xxs_grid[256] = {+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,+};++#define NGRID_IQ2XXS 512+static const uint64_t iq1s_grid[NGRID_IQ2XXS] = {+ 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,+ 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,+ 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,+ 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,+ 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,+ 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,+ 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,+ 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,+ 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,+ 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,+ 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,+ 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,+ 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,+ 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,+ 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,+ 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,+ 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,+ 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,+ 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,+ 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,+ 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,+ 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,+ 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,+ 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,+ 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,+ 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,+ 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,+ 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,+ 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,+ 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,+ 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,+ 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,+ 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,+ 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,+ 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,+ 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,+ 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,+ 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,+ 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,+ 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,+ 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,+ 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,+ 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,+ 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,+ 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,+ 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,+ 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,+ 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,+ 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,+ 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,+ 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,+ 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,+ 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,+ 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,+ 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,+ 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,+ 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,+ 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,+ 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,+ 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,+ 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,+ 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,+ 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,+ 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,+ 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,+ 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,+ 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,+ 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,+ 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,+ 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,+ 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,+ 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,+ 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,+ 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,+ 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,+ 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,+ 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,+ 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,+ 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,+ 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,+ 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,+ 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,+ 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,+ 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,+ 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,+ 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,+ 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,+ 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,+ 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,+ 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,+ 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,+ 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,+ 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,+ 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,+ 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,+ 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,+ 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,+ 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,+ 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,+ 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,+ 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,+ 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,+ 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,+ 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,+ 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,+ 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,+ 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,+ 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,+ 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,+ 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,+ 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,+ 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,+ 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,+ 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,+ 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,+ 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,+ 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,+ 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,+ 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,+ 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,+ 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,+ 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,+ 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,+ 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,+ 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,+ 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,++};+static const uint8_t ksigns_iq2xs[128] = {0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,@@ -3532,6 +3704,101 @@ void dequantize_row_iq2_xs(const block_iq2_xs * restrict x, float * restrict y,}}+// ====================== 3.0625 bpw (de)-quantization++void dequantize_row_iq3_xxs(const block_iq3_xxs * restrict x, float * restrict y, int k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ uint32_t aux32;++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const uint8_t * qs = x[i].qs;+ const uint8_t * scales_and_signs = qs + QK_K/4;++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ memcpy(&aux32, scales_and_signs + 4*ib32, sizeof(uint32_t));+ const float db = d * (0.5f + (aux32 >> 28)) * 0.5f;+ for (int l = 0; l < 4; ++l) {+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + qs[2*l+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + qs[2*l+1]);+ for (int j = 0; j < 4; ++j) {+ y[j+0] = db * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = db * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+ y += 8;+ }+ qs += 8;+ }+ }+}++// ====================== 1.5625 bpw (de)-quantization++void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, int k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++ float db[4];+ uint16_t idx[4];+ //const int8_t * grid[4];++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);+ const uint8_t * sc = x[i].scales;+ const uint8_t * qs = x[i].qs;++ for (int i8 = 0; i8 < QK_K/8; i8 += 4) {+ idx[0] = qs[0] | ((sc[0] & 0x08) << 5);+ idx[1] = qs[1] | ((sc[0] & 0x80) << 1);+ idx[2] = qs[2] | ((sc[1] & 0x08) << 5);+ idx[3] = qs[3] | ((sc[1] & 0x80) << 1);+ //grid[0] = (const int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5)));+ //grid[1] = (const int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1)));+ //grid[2] = (const int8_t *)(iq1s_grid + (qs[2] | ((sc[1] & 0x08) << 5)));+ //grid[3] = (const int8_t *)(iq1s_grid + (qs[3] | ((sc[1] & 0x80) << 1)));+ db[0] = d * (2*(sc[0] & 7) + 1);+ db[1] = d * (2*((sc[0] >> 4) & 7) + 1);+ db[2] = d * (2*(sc[1] & 7) + 1);+ db[3] = d * (2*((sc[1] >> 4) & 7) + 1);+ for (int l = 0; l < 4; ++l) {+ const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);+ for (int j = 0; j < 8; ++j) {+ //y[j] = db[l] * grid[l][j];+ y[j] = db[l] * grid[j];+ }+ y += 8;+ }+ qs += 4;+ sc += 2;+ }+ }+}++static const int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};++void dequantize_row_iq4_nl(const block_iq4_nl * restrict x, float * restrict y, int k) {+ assert(k % QK4_NL == 0);+ const int nb = k / QK4_NL;++ for (int i = 0; i < nb; i++) {++ const uint8_t * qs = x[i].qs;++ const float d = GGML_FP16_TO_FP32(x[i].d);+ for (int j = 0; j < QK4_NL/2; ++j) {+ y[j+ 0] = d * kvalues_iq4nl[qs[j] & 0xf];+ y[j+QK4_NL/2] = d * kvalues_iq4nl[qs[j] >> 4];+ }+ y += QK4_NL;+ qs += QK4_NL/2;+ }+}+//===================================== Q8_K ==============================================void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k) {@@ -3661,13 +3928,13 @@ static inline __m128i get_scale_shuffle(int i) {#undef __AVX2__#endif#undef ggml_vec_dot_q4_0_q8_0-void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q4_0_q8_0_avx2(n, s, vx, vy);+ ggml_vec_dot_q4_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q4_0_q8_0_avx(n, s, vx, vy);+ ggml_vec_dot_q4_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q4_0_q8_0_default(n, s, vx, vy);+ ggml_vec_dot_q4_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -3702,13 +3969,13 @@ void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q4_1_q8_1-void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q4_1_q8_1_avx2(n, s, vx, vy);+ ggml_vec_dot_q4_1_q8_1_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q4_1_q8_1_avx(n, s, vx, vy);+ ggml_vec_dot_q4_1_q8_1_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q4_1_q8_1_default(n, s, vx, vy);+ ggml_vec_dot_q4_1_q8_1_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -3743,13 +4010,13 @@ void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q5_0_q8_0-void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q5_0_q8_0_avx2(n, s, vx, vy);+ ggml_vec_dot_q5_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q5_0_q8_0_avx(n, s, vx, vy);+ ggml_vec_dot_q5_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q5_0_q8_0_default(n, s, vx, vy);+ ggml_vec_dot_q5_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -3784,13 +4051,13 @@ void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q5_1_q8_1-void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q5_1_q8_1_avx2(n, s, vx, vy);+ ggml_vec_dot_q5_1_q8_1_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q5_1_q8_1_avx(n, s, vx, vy);+ ggml_vec_dot_q5_1_q8_1_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q5_1_q8_1_default(n, s, vx, vy);+ ggml_vec_dot_q5_1_q8_1_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -3825,13 +4092,13 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q8_0_q8_0-void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {- ggml_vec_dot_q8_0_q8_0_avx2(n, s, vx, vy);+ ggml_vec_dot_q8_0_q8_0_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q8_0_q8_0_avx(n, s, vx, vy);+ ggml_vec_dot_q8_0_q8_0_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q8_0_q8_0_default(n, s, vx, vy);+ ggml_vec_dot_q8_0_q8_0_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -3868,13 +4135,13 @@ void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q2_K_q8_K-void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q2_K_q8_K_avx2(n, s, vx, vy);+ ggml_vec_dot_q2_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q2_K_q8_K_avx(n, s, vx, vy);+ ggml_vec_dot_q2_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q2_K_q8_K_default(n, s, vx, vy);+ ggml_vec_dot_q2_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -3886,7 +4153,7 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restri#else#error update me-void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const block_q2_K * restrict x = vx;const block_q8_K * restrict y = vy;@@ -4171,13 +4438,13 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q3_K_q8_K-void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q3_K_q8_K_avx2(n, s, vx, vy);+ ggml_vec_dot_q3_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q3_K_q8_K_avx(n, s, vx, vy);+ ggml_vec_dot_q3_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q3_K_q8_K_default(n, s, vx, vy);+ ggml_vec_dot_q3_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -4188,7 +4455,7 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri#else-void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const block_q3_K * restrict x = vx;@@ -4558,13 +4825,13 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q4_K_q8_K-void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q4_K_q8_K_avx2(n, s, vx, vy);+ ggml_vec_dot_q4_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q4_K_q8_K_avx(n, s, vx, vy);+ ggml_vec_dot_q4_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q4_K_q8_K_default(n, s, vx, vy);+ ggml_vec_dot_q4_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -4574,7 +4841,7 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restri#endif#else-void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const block_q4_K * restrict x = vx;@@ -4844,13 +5111,13 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q5_K_q8_K-void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q5_K_q8_K_avx2(n, s, vx, vy);+ ggml_vec_dot_q5_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q5_K_q8_K_avx(n, s, vx, vy);+ ggml_vec_dot_q5_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q5_K_q8_K_default(n, s, vx, vy);+ ggml_vec_dot_q5_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -4861,7 +5128,7 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri#else-void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const block_q5_K * restrict x = vx;@@ -5154,13 +5421,13 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri#undef __AVX2__#endif#undef ggml_vec_dot_q6_K_q8_K-void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_q6_K_q8_K_avx2(n, s, vx, vy);+ ggml_vec_dot_q6_K_q8_K_avx2(n, s, bs, vx, bx, vy, by, nrc);} else if (X86_HAVE(AVX)) {- ggml_vec_dot_q6_K_q8_K_avx(n, s, vx, vy);+ ggml_vec_dot_q6_K_q8_K_avx(n, s, bs, vx, bx, vy, by, nrc);} else {- ggml_vec_dot_q6_K_q8_K_default(n, s, vx, vy);+ ggml_vec_dot_q6_K_q8_K_default(n, s, bs, vx, bx, vy, by, nrc);}}#else@@ -5171,7 +5438,7 @@ void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restri#else-void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const block_q6_K * restrict x = vx;@@ -5501,193 +5768,1493 @@ static const int8_t keven_signs_q2xs[1024] = {1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,};-#ifdef __x86_64__-#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_default-#include "ggml_vec_dot_iq2_xxs_q8_K.inc"-#undef ggml_vec_dot_iq2_xxs_q8_K-#define ggml_vec_dot_iq2_xxs_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_iq2_xxs_q8_K_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_iq2_xxs_q8_K.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_iq2_xxs_q8_K-void ggml_vec_dot_iq2_xxs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_iq2_xxs_q8_K_avx2(n, s, vx, vy);- } else {- ggml_vec_dot_iq2_xxs_q8_K_default(n, s, vx, vy);- }-}+void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq2_xxs * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#if defined(__ARM_NEON)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[4];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ ggml_int8x16x4_t q2u;+ ggml_int8x16x4_t q2s;+ ggml_int8x16x4_t q8b;++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ float sumf1 = 0, sumf2 = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;+ q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 0])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 1])));+ q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 2])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 3])));+ q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 8])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 9])));+ q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[10])), vld1_s8((const void *)(iq2xxs_grid + aux8[11])));+ q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 7) & 127))));+ q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));+ q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 7) & 127))));+ q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 21) & 127))));+ q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);+ q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);+ q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);+ q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]), q2u.val[1], q8b.val[1]);+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]), q2u.val[3], q8b.val[3]);+ sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28));+ sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[3] >> 28));+ }+ sumf += d*(sumf1 + sumf2);+ }+ *s = 0.25f * sumf;++#elif defined(__AVX2__)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[4];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;+ const __m256i q2_1 = _mm256_set_epi64x(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);+ const __m256i q2_2 = _mm256_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);+ const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],+ signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const uint16_t ls1 = aux32[1] >> 28;+ const uint16_t ls2 = aux32[3] >> 28;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));+ sumi1 = _mm256_add_epi32(sumi1, p1);+ sumi2 = _mm256_add_epi32(sumi2, p2);+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);+#else-#define static-#include "ggml_vec_dot_iq2_xxs_q8_K.inc"-#undef static-#endif-#ifdef __x86_64__-#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_default-#include "ggml_vec_dot_iq2_xs_q8_K.inc"-#undef ggml_vec_dot_iq2_xs_q8_K-#define ggml_vec_dot_iq2_xs_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_iq2_xs_q8_K_avx2-#ifndef __AVX2__-#define __UNDEF_AVX2__-#define __AVX2__-#endif-#include "ggml_vec_dot_iq2_xs_q8_K.inc"-#ifdef __UNDEF_AVX2__-#undef __UNDEF_AVX2__-#undef __AVX2__-#endif-#undef ggml_vec_dot_iq2_xs_q8_K-void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {- ggml_vec_dot_iq2_xs_q8_K_avx2(n, s, vx, vy);- } else {- ggml_vec_dot_iq2_xs_q8_K_default(n, s, vx, vy);+ uint32_t aux32[2];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ memcpy(aux32, q2, 2*sizeof(uint32_t));+ q2 += 4;+ const uint32_t ls = 2*(aux32[1] >> 28) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);+ const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls;+ }+ sumf += d * bsum;}-}-#else-#define static-#include "ggml_vec_dot_iq2_xs_q8_K.inc"-#undef static+ *s = 0.125f * sumf;#endif+}-// ================================ IQ2 quantization =============================================+void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);-typedef struct {- uint64_t * grid;- int * map;- uint16_t * neighbours;-} iq2_entry_t;+ const block_iq2_xs * restrict x = vx;+ const block_q8_K * restrict y = vy;-static iq2_entry_t iq2_data[2] = {- {NULL, NULL, NULL},- {NULL, NULL, NULL},-};+ const int nb = n / QK_K;-static inline int iq2_data_index(int grid_size) {- GGML_ASSERT(grid_size == 256 || grid_size == 512);- return grid_size == 256 ? 0 : 1;-}+#if defined(__ARM_NEON)-static int iq2_compare_func(const void * left, const void * right) {- const int * l = (const int *)left;- const int * r = (const int *)right;- return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;-}+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;-void iq2xs_init_impl(int grid_size) {- const int gindex = iq2_data_index(grid_size);- if (iq2_data[gindex].grid) {- return;- }- static const uint16_t kgrid_256[256] = {- 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97,- 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642,- 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288,- 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113,- 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240,- 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400,- 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260,- 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872,- 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516,- 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561,- 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488,- 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545,- 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874,- 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856,- 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142,- 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268,+ ggml_int8x16x4_t q2u;+ ggml_int8x16x4_t q2s;+ ggml_int8x16x4_t q8b;++ int32x4x4_t scales32;++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ const uint8x8_t scales8 = vld1_u8(x[i].scales);+ const uint8x8_t scales_l = vand_u8(scales8, vdup_n_u8(0xf));+ const uint8x8_t scales_h = vshr_n_u8(scales8, 4);+ uint8x16_t scales = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h));+ scales = vaddq_u8(vshlq_n_u8(scales, 1), vdupq_n_u8(1));+ const uint16x8_t scales1 = vmovl_u8(vget_low_u8(scales));+ const uint16x8_t scales2 = vmovl_u8(vget_high_u8(scales));+ scales32.val[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales1)));+ scales32.val[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales1)));+ scales32.val[2] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales2)));+ scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2)));+ int32x4_t sumi = vdupq_n_s32(0);+ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;+ q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511))));+ q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511))));+ q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511))));+ q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[6] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[7] & 511))));+ q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[0] >> 9))), vld1_s8((const void *)(signs64 + (q2[1] >> 9))));+ q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[2] >> 9))), vld1_s8((const void *)(signs64 + (q2[3] >> 9))));+ q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[4] >> 9))), vld1_s8((const void *)(signs64 + (q2[5] >> 9))));+ q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[6] >> 9))), vld1_s8((const void *)(signs64 + (q2[7] >> 9))));+ q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);+ q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);+ q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);+ q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);+ const int32x4_t p1 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]);+ const int32x4_t p2 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[1], q8b.val[1]);+ const int32x4_t p3 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]);+ const int32x4_t p4 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[3], q8b.val[3]);+ const int32x4_t p = vpaddq_s32(vpaddq_s32(p1, p2), vpaddq_s32(p3, p4));+ sumi = vmlaq_s32(sumi, p, scales32.val[ib64]);+ q2 += 8;+ }+ sumf += d*vaddvq_s32(sumi);+ }+ *s = 0.125f * sumf;++#elif defined(__AVX2__)++ const __m128i m4 = _mm_set1_epi8(0xf);+ const __m128i m1 = _mm_set1_epi8(1);+ const __m256i m511 = _mm256_set1_epi16(511);+ const __m256i mone = _mm256_set1_epi8(1);++ static const uint8_t k_bit_helper[32] = {+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,};- static const uint16_t kgrid_512[512] = {- 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,- 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257,- 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340,- 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597,- 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096,- 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348,- 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065,- 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441,- 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160,- 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372,- 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125,- 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652,- 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197,- 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549,- 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894,- 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388,- 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480,- 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773,- 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473,- 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436,- 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497,- 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162,- 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528,- 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745,- 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234,- 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025,- 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810,- 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984,- 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462,- 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960,- 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048,- 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690,+ static const char block_sign_shuffle_mask_1[32] = {+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,+ 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,+ };+ static const char block_sign_shuffle_mask_2[32] = {+ 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,+ };+ static const uint8_t bit_selector_mask_bytes[32] = {+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};- const int kmap_size = 43692;- const int nwant = 2;- const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512;- uint64_t * kgrid_q2xs;- int * kmap_q2xs;- uint16_t * kneighbors_q2xs;- printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);- uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t));- for (int k = 0; k < grid_size; ++k) {- int8_t * pos = (int8_t *)(the_grid + k);- for (int i = 0; i < 8; ++i) {- int l = (kgrid[k] >> 2*i) & 0x3;- pos[i] = 2*l + 1;- }- }- kgrid_q2xs = the_grid;- iq2_data[gindex].grid = the_grid;- kmap_q2xs = (int *)malloc(kmap_size*sizeof(int));- iq2_data[gindex].map = kmap_q2xs;- for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1;+ const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper);+ const __m256i bit_selector_mask = _mm256_loadu_si256((const __m256i*)bit_selector_mask_bytes);+ const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1);+ const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2);+uint64_t aux64;- uint8_t * aux8 = (uint8_t *)&aux64;- for (int i = 0; i < grid_size; ++i) {- aux64 = kgrid_q2xs[i];- uint16_t index = 0;- for (int k=0; k<8; ++k) {- uint16_t q = (aux8[k] - 1)/2;- index |= (q << 2*k);++ // somewhat hacky, but gives a significant boost in performance+ __m256i aux_gindex;+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ memcpy(&aux64, x[i].scales, 8);+ __m128i stmp = _mm_set1_epi64x(aux64);+ stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));+ const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);++ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {++ const __m256i q2_data = _mm256_loadu_si256((const __m256i*)q2); q2 += 16;+ aux_gindex = _mm256_and_si256(q2_data, m511);++ const __m256i partial_sign_bits = _mm256_srli_epi16(q2_data, 9);+ const __m256i partial_sign_bits_upper = _mm256_srli_epi16(q2_data, 13);+ const __m256i partial_sign_bits_for_counting = _mm256_xor_si256(partial_sign_bits, partial_sign_bits_upper);++ const __m256i odd_bits = _mm256_shuffle_epi8(bit_helper, partial_sign_bits_for_counting);+ const __m256i full_sign_bits = _mm256_or_si256(partial_sign_bits, odd_bits);++ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_4 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;++ const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],+ iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);+ const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],+ iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);+ const __m256i q2_3 = _mm256_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],+ iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);+ const __m256i q2_4 = _mm256_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],+ iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);++ const __m128i full_signs_l = _mm256_castsi256_si128(full_sign_bits);+ const __m128i full_signs_h = _mm256_extractf128_si256(full_sign_bits, 1);+ const __m256i full_signs_1 = _mm256_set_m128i(full_signs_l, full_signs_l);+ const __m256i full_signs_2 = _mm256_set_m128i(full_signs_h, full_signs_h);++ __m256i signs;+ signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_1);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone));++ signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_2);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone));++ signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_1);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_3 = _mm256_sign_epi8(q8_3, _mm256_or_si256(signs, mone));++ signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_2);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_4 = _mm256_sign_epi8(q8_4, _mm256_or_si256(signs, mone));++ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const __m256i dot3 = _mm256_maddubs_epi16(q2_3, q8s_3);+ const __m256i dot4 = _mm256_maddubs_epi16(q2_4, q8s_4);++ const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0)));+ const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1)));+ const __m256i sc3 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2)));+ const __m256i sc4 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3)));++ sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1));+ sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2));+ sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot3, sc3));+ sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot4, sc4));}- kmap_q2xs[index] = i;++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);+}- int8_t pos[8];- int * dist2 = (int *)malloc(2*grid_size*sizeof(int));- int num_neighbors = 0, num_not_in_map = 0;- for (int i = 0; i < kmap_size; ++i) {- if (kmap_q2xs[i] >= 0) continue;- ++num_not_in_map;- for (int k = 0; k < 8; ++k) {- int l = (i >> 2*k) & 0x3;- pos[k] = 2*l + 1;- }- for (int j = 0; j < grid_size; ++j) {++ *s = 0.125f * hsum_float_8(accumf);++#else++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const uint8_t * restrict sc = x[i].scales;+ const int8_t * restrict q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;+ const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 2; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls1;+ sumi = 0;+ for (int l = 2; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls2;+ q2 += 4;+ }+ sumf += d * bsum;+ }+ *s = 0.125f * sumf;+#endif+}++void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq3_xxs * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#if defined(__ARM_NEON)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[2];++ ggml_int8x16x4_t q3s;+ ggml_int8x16x4_t q8b;++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict gas = x[i].qs + QK_K/4;+ const int8_t * restrict q8 = y[i].qs;+ float sumf1 = 0, sumf2 = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;+ memcpy(aux32, gas, 2*sizeof(uint32_t)); gas += 2*sizeof(uint32_t);+ const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]);+ const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]);+ const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]);+ const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]);+ q3 += 16;+ q3s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 7) & 127))));+ q3s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 21) & 127))));+ q3s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 7) & 127))));+ q3s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));+ q3s.val[0] = vmulq_s8(q3s.val[0], vreinterpretq_s8_u32(aux32x4_0));+ q3s.val[1] = vmulq_s8(q3s.val[1], vreinterpretq_s8_u32(aux32x4_1));+ q3s.val[2] = vmulq_s8(q3s.val[2], vreinterpretq_s8_u32(aux32x4_2));+ q3s.val[3] = vmulq_s8(q3s.val[3], vreinterpretq_s8_u32(aux32x4_3));+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);+ sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28));+ sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[1] >> 28));+ }+ sumf += d*(sumf1 + sumf2);+ }+ *s = 0.5f * sumf;++#elif defined(__AVX2__)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[2];++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict gas = x[i].qs + QK_K/4;+ const int8_t * restrict q8 = y[i].qs;+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q2_1 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ q3 += 8;+ const __m256i q2_2 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ q3 += 8;+ memcpy(aux32, gas, 8); gas += 8;+ const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],+ signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);+ const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const uint16_t ls1 = aux32[0] >> 28;+ const uint16_t ls2 = aux32[1] >> 28;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));+ sumi1 = _mm256_add_epi32(sumi1, p1);+ sumi2 = _mm256_add_epi32(sumi2, p2);+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.25f * hsum_float_8(accumf);++#else++ uint32_t aux32;++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict gas = x[i].qs + QK_K/4;+ const int8_t * restrict q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ memcpy(&aux32, gas, sizeof(uint32_t)); gas += sizeof(uint32_t);+ const uint32_t ls = 2*(aux32 >> 28) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*l+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*l+1]);+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];+ for (int j = 0; j < 4; ++j) {+ sumi += grid1[j] * q8[j+0] * (signs & kmask_iq2xs[j+0] ? -1 : 1);+ sumi += grid2[j] * q8[j+4] * (signs & kmask_iq2xs[j+4] ? -1 : 1);+ }+ q8 += 8;+ }+ q3 += 8;+ bsum += sumi * ls;+ }+ sumf += d * bsum;+ }+ *s = 0.25f * sumf;+#endif+}++#ifdef __AVX2__+static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {+ const __m256i ax = _mm256_sign_epi8(x, x);+ const __m256i sy = _mm256_sign_epi8(y, x);+ return _mm256_maddubs_epi16(ax, sy);+}+#endif++void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_iq1_s * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#if defined __ARM_NEON++ const uint8x16_t m8 = vdupq_n_u8(0x08);+ const uint8x16_t m7 = vdupq_n_u8(0x07);+ const uint8x16_t m1 = vdupq_n_u8(0x01);+ const int32x4_t vzero = vdupq_n_s32(0);++ uint16_t gindex[8];+ uint16x8x2_t vindex;+ int8x16x4_t q1b;+ int8x16x4_t q8b;+ uint16x8x4_t scales;+ int32x4x2_t sumi;+ int32x4x2_t dotq;++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint8_t * sc = x[i].scales;++ sumi.val[0] = sumi.val[1] = vzero;++ for (int i128 = 0; i128 < QK_K/128; ++i128) {+ const uint8x16_t ql = vld1q_u8(qs); qs += 16;+ const uint8x8_t tm1 = vld1_u8 (sc); sc += 8;+ const uint8x8_t tm2 = vshr_n_u8(tm1, 4);+ const uint8x16_t qh = vcombine_u8(vzip1_u8(tm1, tm2), vzip2_u8(tm1, tm2));+ const uint8x16_t hbit = vandq_u8(qh, m8);+ vindex.val[0] = vorrq_u16(vmovl_u8(vget_low_u8 (ql)), vshlq_n_u16(vmovl_u8(vget_low_u8 (hbit)), 5));+ vindex.val[1] = vorrq_u16(vmovl_u8(vget_high_u8(ql)), vshlq_n_u16(vmovl_u8(vget_high_u8(hbit)), 5));+ const uint8x16_t scales8 = vorrq_u8(vshlq_n_u8(vandq_u8(qh, m7), 1), m1);+ scales.val[0] = vmovl_u8(vget_low_u8 (scales8));+ scales.val[1] = vmovl_u8(vget_high_u8 (scales8));++ for (int l = 0; l < 2; ++l) {+ vst1q_u16(gindex+0, vindex.val[l]);+ q1b.val[0] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[0])), vld1_s8((const void *)(iq1s_grid+gindex[1])));+ q1b.val[1] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[2])), vld1_s8((const void *)(iq1s_grid+gindex[3])));+ q1b.val[2] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[4])), vld1_s8((const void *)(iq1s_grid+gindex[5])));+ q1b.val[3] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[6])), vld1_s8((const void *)(iq1s_grid+gindex[7])));+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;++ dotq.val[0] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[0], q8b.val[0]), ggml_vdotq_s32(vzero, q1b.val[1], q8b.val[1]));+ dotq.val[1] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(vzero, q1b.val[3], q8b.val[3]));++ sumi.val[0] = vmlaq_s32(sumi.val[0], dotq.val[0], vreinterpretq_s32_u32(vmovl_u16(vget_low_u16 (scales.val[l]))));+ sumi.val[1] = vmlaq_s32(sumi.val[1], dotq.val[1], vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales.val[l]))));+ }+ }++ sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * vaddvq_s32(vaddq_s32(sumi.val[0], sumi.val[1]));+ }++ *s = sumf;++#elif defined __AVX2__++ const __m128i m8 = _mm_set1_epi8(0x08);+ const __m128i m7 = _mm_set1_epi8(0x07);+ const __m128i m1 = _mm_set1_epi8(0x01);+ const __m128i shuffle_h = _mm_set_epi8(15, 7, 14, 6, 13, 5, 12, 4, 11, 3, 10, 2, 9, 1, 8, 0);+ const __m128i shuffle_s[4] = {+ _mm_set_epi32(0x03030303, 0x02020202, 0x01010101, 0x00000000),+ _mm_set_epi32(0x07070707, 0x06060606, 0x05050505, 0x04040404),+ _mm_set_epi32(0x0b0b0b0b, 0x0a0a0a0a, 0x09090909, 0x08080808),+ _mm_set_epi32(0x0f0f0f0f, 0x0e0e0e0e, 0x0d0d0d0d, 0x0c0c0c0c)+ };++ uint64_t aux64;++ __m256i v_gindex;+ const uint16_t * gindex = (const uint16_t *)&v_gindex;++ __m256 accum = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint8_t * sc = x[i].scales;++ __m256i sumi = _mm256_setzero_si256();+ for (int i128 = 0; i128 < QK_K/128; ++i128) {+ const __m128i ql = _mm_loadu_si128((const __m128i*)qs); qs += 16;+ memcpy(&aux64, sc, 8); sc += 8;+ const __m128i qh = _mm_shuffle_epi8(_mm_set_epi64x(aux64 >> 4, aux64), shuffle_h);+ const __m256i hbit = _mm256_cvtepu8_epi16(_mm_and_si128(qh, m8));+ v_gindex = _mm256_or_si256(_mm256_cvtepu8_epi16(ql), _mm256_slli_epi16(hbit, 5));+ const __m128i scales = _mm_or_si128(_mm_slli_epi16(_mm_and_si128(qh, m7), 1), m1);++ for (int i32 = 0; i32 < 4; ++i32) {+ const __m256i q8b = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q1b = _mm256_set_epi64x(iq1s_grid[gindex[4*i32+3]], iq1s_grid[gindex[4*i32+2]],+ iq1s_grid[gindex[4*i32+1]], iq1s_grid[gindex[4*i32+0]]);+ const __m256i dot = mul_add_epi8(q1b, q8b);+ const __m256i s16 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, shuffle_s[i32]));+ const __m256i p = _mm256_madd_epi16(s16, dot);+ sumi = _mm256_add_epi32(sumi, p);+ }++ }++ accum = _mm256_fmadd_ps(_mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(x[i].d)), _mm256_cvtepi32_ps(sumi), accum);++ }++ *s = hsum_float_8(accum);++#else++ int db[4];+ uint16_t idx[4];++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint8_t * sc = x[i].scales;++ int sumi = 0;+ for (int i32 = 0; i32 < QK_K/32; ++i32) {+ idx[0] = qs[0] | ((sc[0] & 0x08) << 5);+ idx[1] = qs[1] | ((sc[0] & 0x80) << 1);+ idx[2] = qs[2] | ((sc[1] & 0x08) << 5);+ idx[3] = qs[3] | ((sc[1] & 0x80) << 1);+ db[0] = (2*(sc[0] & 7) + 1);+ db[1] = (2*((sc[0] >> 4) & 7) + 1);+ db[2] = (2*(sc[1] & 7) + 1);+ db[3] = (2*((sc[1] >> 4) & 7) + 1);+ for (int l = 0; l < 4; ++l) {+ const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);+ int suml = 0;+ for (int j = 0; j < 8; ++j) suml += q8[j] * grid[j];+ sumi += db[l] * suml;+ q8 += 8;+ }+ qs += 4;+ sc += 2;+ }++ sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * sumi;+ }++ *s = sumf;++#endif+}++void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ assert(n % QK4_NL == 0);+ static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same");++ const block_iq4_nl * restrict x = vx;+ const block_q8_0 * restrict y = vy;++ const int nb = n / QK4_NL;++#if defined __ARM_NEON+ const int8x16_t values = vld1q_s8(kvalues_iq4nl);+ const uint8x16_t m4b = vdupq_n_u8(0x0f);+ uint8x16x2_t q4bits;+ int8x16x4_t q4b;+ int8x16x4_t q8b;+ int32x4_t prod_1, prod_2;++ float sumf = 0;++ for (int ib = 0; ib < nb; ib += 2) {++ q4bits.val[0] = vld1q_u8(x[ib+0].qs);+ q4bits.val[1] = vld1q_u8(x[ib+1].qs);+ q8b.val[0] = vld1q_s8(y[ib+0].qs);+ q8b.val[1] = vld1q_s8(y[ib+0].qs + 16);+ q8b.val[2] = vld1q_s8(y[ib+1].qs);+ q8b.val[3] = vld1q_s8(y[ib+1].qs + 16);++ q4b.val[0] = vqtbl1q_s8(values, vandq_u8(q4bits.val[0], m4b));+ q4b.val[1] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));+ q4b.val[2] = vqtbl1q_s8(values, vandq_u8(q4bits.val[1], m4b));+ q4b.val[3] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));++ prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);+ prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);++ sumf += (float)x[ib+0].d * (float)y[ib+0].d * vaddvq_s32(prod_1) + (float)x[ib+1].d * (float)y[ib+1].d * vaddvq_s32(prod_2);++ }++ *s = sumf;++#elif defined __AVX2__++ const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);+ const __m128i m4b = _mm_set1_epi8(0x0f);+ const __m256i mone = _mm256_set1_epi16(1);++ __m256 accum1 = _mm256_setzero_ps();+ __m256 accum2 = _mm256_setzero_ps();+ for (int ib = 0; ib < nb; ib += 2) {+ const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[0].qs);+ const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[1].qs);+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[0].qs);+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[1].qs);+ const __m256i q4b_1 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));+ const __m256i q4b_2 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);+ const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);+ const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);+ accum1 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[0].d)*GGML_FP16_TO_FP32(x[0].d)),+ _mm256_cvtepi32_ps(p_1), accum1);+ accum2 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(y[1].d)*GGML_FP16_TO_FP32(x[1].d)),+ _mm256_cvtepi32_ps(p_2), accum2);++ y += 2;+ x += 2;+ }++ *s = hsum_float_8(_mm256_add_ps(accum1, accum2));++#else+ float sumf = 0;+ for (int ib = 0; ib < nb; ++ib) {+ const float d = GGML_FP16_TO_FP32(y[ib].d)*GGML_FP16_TO_FP32(x[ib].d);+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < QK4_NL/2; ++j) {+ sumi1 += y[ib].qs[j+ 0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];+ sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >> 4];+ }+ sumf += d * (sumi1 + sumi2);+ }+ *s = sumf;+#endif+}++// ================================ IQ2 quantization =============================================++typedef struct {+ uint64_t * grid;+ int * map;+ uint16_t * neighbours;+} iq2_entry_t;++static iq2_entry_t iq2_data[3] = {+ {NULL, NULL, NULL},+ {NULL, NULL, NULL},+ {NULL, NULL, NULL},+};++static inline int iq2_data_index(enum ggml_type type) {+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S);+ return type == GGML_TYPE_IQ2_XXS ? 0 :+ type == GGML_TYPE_IQ2_XS ? 1 : 2;+}++static inline int iq2_grid_size(enum ggml_type type) {+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S);+ return type == GGML_TYPE_IQ2_XXS ? 256 :+ type == GGML_TYPE_IQ2_XS ? 512 : 512;+}++static int iq2_compare_func(const void * left, const void * right) {+ const int * l = (const int *)left;+ const int * r = (const int *)right;+ return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;+}++void iq2xs_init_impl(enum ggml_type type) {+ const int gindex = iq2_data_index(type);+ const int grid_size = iq2_grid_size(type);+ if (iq2_data[gindex].grid) {+ return;+ }+ static const uint16_t kgrid_2bit_256[256] = {+ 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97,+ 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642,+ 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288,+ 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113,+ 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240,+ 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400,+ 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260,+ 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872,+ 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516,+ 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561,+ 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488,+ 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545,+ 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874,+ 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856,+ 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142,+ 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268,+ };+ static const uint16_t kgrid_2bit_512[512] = {+ 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,+ 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257,+ 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340,+ 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597,+ 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096,+ 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348,+ 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065,+ 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441,+ 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160,+ 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372,+ 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125,+ 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652,+ 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197,+ 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549,+ 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894,+ 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388,+ 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480,+ 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773,+ 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473,+ 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436,+ 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497,+ 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162,+ 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528,+ 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745,+ 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234,+ 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025,+ 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810,+ 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984,+ 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462,+ 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960,+ 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048,+ 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690,+ };+ static const uint16_t kgrid_1bit_512[512] = {+ 10, 33, 41, 85, 132, 134, 160, 162, 277, 337, 340, 345, 357, 405, 516, 545,+ 553, 598, 641, 650, 681, 1042, 1044, 1097, 1169, 1176, 1320, 1345, 1365, 1378, 1434, 1444,+ 1545, 1617, 1642, 1685, 2053, 2080, 2089, 2133, 2176, 2182, 2208, 2214, 2306, 2384, 2393, 2440,+ 2453, 2581, 2664, 2690, 2721, 4117, 4161, 4182, 4184, 4261, 4357, 4369, 4372, 4377, 4390, 4422,+ 4432, 4437, 4449, 4457, 4485, 4497, 4505, 4629, 4677, 4696, 4774, 5205, 5217, 5225, 5386, 5397,+ 5409, 5445, 5457, 5460, 5461, 5462, 5465, 5472, 5477, 5525, 5545, 5650, 5668, 5717, 5729, 5769,+ 5777, 6212, 6234, 6244, 6293, 6424, 6482, 6485, 6502, 6505, 6529, 6538, 6565, 6656, 6682, 6788,+ 6806, 6820, 8218, 8224, 8226, 8232, 8277, 8326, 8354, 8469, 8521, 8530, 8549, 8596, 8737, 8794,+ 9221, 9253, 9348, 9369, 9380, 9474, 9557, 9633, 9732, 9753, 9793, 9830, 9862, 9880, 10240, 10272,+ 10282, 10321, 10406, 10517, 10530, 10566, 10585, 10645, 10896, 16466, 16468, 16473, 16485, 16646, 16660, 16665,+ 16725, 16793, 16806, 16914, 16969, 16977, 16996, 17028, 17057, 17408, 17416, 17434, 17493, 17512, 17578, 17685,+ 17696, 17733, 17745, 17748, 17749, 17750, 17753, 17765, 17794, 17813, 17946, 17984, 18005, 18072, 18453, 18529,+ 18569, 18722, 18756, 18762, 18773, 18794, 18833, 18853, 18945, 19026, 19033, 19077, 20489, 20497, 20500, 20517,+ 20565, 20586, 20610, 20633, 20757, 20769, 20776, 20805, 20817, 20820, 20821, 20822, 20825, 20837, 20864, 20872,+ 20885, 20896, 21002, 21029, 21077, 21146, 21510, 21525, 21573, 21585, 21588, 21589, 21590, 21593, 21605, 21653,+ 21665, 21765, 21777, 21780, 21781, 21782, 21785, 21797, 21825, 21828, 21829, 21830, 21833, 21840, 21841, 21842,+ 21844, 21846, 21848, 21849, 21850, 21857, 21860, 21861, 21862, 21865, 21893, 21905, 21908, 21909, 21910, 21913,+ 21925, 22024, 22037, 22085, 22097, 22100, 22101, 22102, 22105, 22117, 22165, 22545, 22566, 22568, 22594, 22608,+ 22613, 22676, 22697, 22793, 22805, 22853, 22865, 22868, 22869, 22870, 22873, 22885, 22933, 22946, 23046, 23072,+ 23125, 23209, 24597, 24640, 24665, 24673, 24725, 24833, 24840, 24869, 24917, 24934, 24965, 25001, 25108, 25110,+ 25152, 25184, 25192, 25234, 25616, 25618, 25625, 25685, 25704, 25738, 25744, 25770, 25877, 25897, 25925, 25937,+ 25940, 25941, 25942, 25945, 25957, 25986, 26005, 26186, 26197, 26276, 26632, 26634, 26725, 26757, 26770, 26885,+ 26965, 26976, 26986, 27032, 27153, 27174, 27200, 27208, 27240, 27269, 27282, 27290, 32778, 32800, 32802, 32808,+ 32810, 32853, 32904, 32922, 32930, 32932, 33105, 33110, 33112, 33125, 33157, 33280, 33288, 33301, 33312, 33320,+ 33424, 33797, 33829, 33858, 34068, 34133, 34146, 34176, 34217, 34306, 34342, 34441, 34454, 34468, 34832, 34918,+ 34965, 34984, 35094, 35137, 35161, 35208, 35232, 35332, 35338, 35368, 35429, 36932, 36934, 36953, 37009, 37125,+ 37136, 37138, 37145, 37157, 37205, 37220, 37258, 37290, 37444, 37446, 37465, 37478, 37525, 37905, 37968, 37973,+ 38040, 38054, 38145, 38154, 38165, 38180, 38186, 38213, 38225, 38228, 38229, 38230, 38233, 38245, 38293, 38485,+ 38504, 38530, 38938, 38985, 38993, 39012, 39040, 39173, 39192, 39253, 39265, 39301, 39316, 39322, 39442, 39497,+ 39504, 39590, 40970, 40984, 40992, 41002, 41045, 41120, 41128, 41237, 41289, 41297, 41317, 41364, 41366, 41514,+ 41557, 41633, 41989, 42021, 42056, 42068, 42074, 42113, 42242, 42265, 42274, 42325, 42340, 42402, 42501, 42512,+ 42533, 42624, 42632, 42666, 43040, 43093, 43106, 43168, 43176, 43264, 43286, 43345, 43429, 43590, 43618, 43680,+ };++ const int kmap_size = 43692;+ const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2;+ const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 :+ type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : kgrid_1bit_512;+ uint64_t * kgrid_q2xs;+ int * kmap_q2xs;+ uint16_t * kneighbors_q2xs;++ printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);+ uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t));+ for (int k = 0; k < grid_size; ++k) {+ int8_t * pos = (int8_t *)(the_grid + k);+ for (int i = 0; i < 8; ++i) {+ int l = (kgrid[k] >> 2*i) & 0x3;+ pos[i] = 2*l + 1;+ }+ }+ kgrid_q2xs = the_grid;+ iq2_data[gindex].grid = the_grid;+ kmap_q2xs = (int *)malloc(kmap_size*sizeof(int));+ iq2_data[gindex].map = kmap_q2xs;+ for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1;+ uint64_t aux64;+ uint8_t * aux8 = (uint8_t *)&aux64;+ for (int i = 0; i < grid_size; ++i) {+ aux64 = kgrid_q2xs[i];+ uint16_t index = 0;+ for (int k=0; k<8; ++k) {+ uint16_t q = (aux8[k] - 1)/2;+ index |= (q << 2*k);+ }+ kmap_q2xs[index] = i;+ }+ int8_t pos[8];+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));+ int num_neighbors = 0, num_not_in_map = 0;+ for (int i = 0; i < kmap_size; ++i) {+ if (kmap_q2xs[i] >= 0) continue;+ ++num_not_in_map;+ for (int k = 0; k < 8; ++k) {+ int l = (i >> 2*k) & 0x3;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);+ int d2 = 0;+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);+ int n = 0; int d2 = dist2[0];+ int nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ ++n;+ }+ num_neighbors += n;+ }+ printf("%s: %d neighbours in total\n", __func__, num_neighbors);+ kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));+ iq2_data[gindex].neighbours = kneighbors_q2xs;+ int counter = 0;+ for (int i = 0; i < kmap_size; ++i) {+ if (kmap_q2xs[i] >= 0) continue;+ for (int k = 0; k < 8; ++k) {+ int l = (i >> 2*k) & 0x3;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);int d2 = 0;- for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);+ kmap_q2xs[i] = -(counter + 1);+ int d2 = dist2[0];+ uint16_t * start = &kneighbors_q2xs[counter++];+ int n = 0, nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ kneighbors_q2xs[counter++] = dist2[2*j+1];+ ++n;+ }+ *start = n;+ }+ free(dist2);+}++void iq2xs_free_impl(enum ggml_type type) {+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S);+ const int gindex = iq2_data_index(type);+ if (iq2_data[gindex].grid) {+ free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL;+ free(iq2_data[gindex].map); iq2_data[gindex].map = NULL;+ free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL;+ }+}++static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid,+ const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_d2 = FLT_MAX;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float d2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = pg[i];+ float diff = scale*q - xval[i];+ d2 += weight[i]*diff*diff;+ }+ if (d2 < best_d2) {+ best_d2 = d2; grid_index = neighbours[j];+ }+ }+ GGML_ASSERT(grid_index >= 0);+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}++static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 3;++ const int nbl = n/256;++ block_iq2_xxs * y = vy;++ float scales[QK_K/32];+ float weight[32];+ float xval[32];+ int8_t L[32];+ int8_t Laux[32];+ float waux[32];+ uint8_t block_signs[4];+ uint32_t q2[2*(QK_K/32)];++ for (int ibl = 0; ibl < nbl; ++ibl) {++ y[ibl].d = GGML_FP32_TO_FP16(0.f);+ memset(q2, 0, QK_K/4);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = sumx2/QK_K;++ for (int ib = 0; ib < QK_K/32; ++ib) {+ const float * xb = xbl + 32*ib;+ const float * qw = quant_weights + QK_K*ibl + 32*ib;+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 4; ++k) {+ int nflip = 0;+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);+ }+ }+ if (nflip%2) {+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];+ for (int i = 1; i < 8; ++i) {+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];+ if (ax < min) {+ min = ax; imin = i;+ }+ }+ xval[8*k+imin] = -xval[8*k+imin];+ s ^= (1 << imin);+ }+ block_signs[k] = s & 127;+ }+ float max = xval[0];+ for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);+ if (!max) {+ scales[ib] = 0;+ memset(L, 0, 32);+ continue;+ }+ float scale = make_qp_quants(32, kMaxQ+1, xval, (uint8_t*)L, weight);+ float eff_max = scale*kMaxQ;+ float best = 0;+ for (int is = -6; is <= 6; ++is) {+ float id = (2*kMaxQ-1+is*0.1f)/eff_max;+ float this_scale = 1/id;+ for (int k = 0; k < 4; ++k) {+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ memcpy(L, Laux, 32);+ }+ }+ if (scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 4; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 2*i);+ }+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ }+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);+ for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)+ // and correspondingly flip quant signs.+ scale = -scale;+ for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;+ }+ for (int k = 0; k < 4; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ printf("\n");+ GGML_ASSERT(false);+ }+ q2[2*ib+0] |= (grid_index << 8*k);+ q2[2*ib+1] |= (block_signs[k] << 7*k);+ }+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ memset(y[ibl].qs, 0, QK_K/4);+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d);+ float id = 1/d;+ for (int ib = 0; ib < QK_K/32; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ q2[2*ib+1] |= ((uint32_t)l << 28);+ }+ memcpy(y[ibl].qs, q2, QK_K/4);+ }+}++static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ2_XS);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 3;++ const int nbl = n/256;++ block_iq2_xs * y = vy;++ float scales[QK_K/16];+ float weight[16];+ float xval[16];+ int8_t L[16];+ int8_t Laux[16];+ float waux[16];+ bool is_on_grid[2];+ bool is_on_grid_aux[2];+ uint8_t block_signs[2];+ uint16_t q2[2*(QK_K/16)];++ for (int ibl = 0; ibl < nbl; ++ibl) {++ y[ibl].d = GGML_FP32_TO_FP16(0.f);+ memset(q2, 0, QK_K/4);+ memset(y[ibl].scales, 0, QK_K/32);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = sumx2/QK_K;++ for (int ib = 0; ib < QK_K/16; ++ib) {+ const float * xb = xbl + 16*ib;+ const float * qw = quant_weights + QK_K*ibl + 16*ib;+ for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 2; ++k) {+ int nflip = 0;+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);+ }+ }+ if (nflip%2) {+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];+ for (int i = 1; i < 8; ++i) {+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];+ if (ax < min) {+ min = ax; imin = i;+ }+ }+ xval[8*k+imin] = -xval[8*k+imin];+ s ^= (1 << imin);+ }+ block_signs[k] = s & 127;+ }+ float max = xval[0];+ for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);+ if (!max) {+ scales[ib] = 0;+ memset(L, 0, 16);+ continue;+ }+ float best = 0;+ float scale = max/(2*kMaxQ-1);+ is_on_grid[0] = is_on_grid[1] = true;+ for (int is = -9; is <= 9; ++is) {+ float id = (2*kMaxQ-1+is*0.1f)/max;+ float this_scale = 1/id;+ for (int k = 0; k < 2; ++k) {+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ is_on_grid_aux[k] = true;+ if (grid_index < 0) {+ is_on_grid_aux[k] = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ for (int i = 0; i < 16; ++i) L[i] = Laux[i];+ for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];+ }+ }+ int n_not_ongrid = 0;+ for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ if (n_not_ongrid > 0 && scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 2; ++k) {+ if (is_on_grid[k]) continue;+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 2*i);+ L[8*k + i] = l;+ }+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ scale = -scale;+ for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127;+ }+ for (int k = 0; k < 2; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ printf("\n");+ GGML_ASSERT(false);+ }+ q2[2*ib+k] = grid_index | (block_signs[k] << 9);+ }+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ memset(y[ibl].qs, 0, QK_K/4);+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d);+ float id = 1/d;+ for (int ib = 0; ib < QK_K/16; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ if (ib%2 == 0) y[ibl].scales[ib/2] = l;+ else y[ibl].scales[ib/2] |= (l << 4);+ }+ memcpy(y[ibl].qs, q2, QK_K/4);++ }+}++size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ GGML_ASSERT(n_per_row%QK_K == 0);+ int nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq2_xxs);+ }+ return nrow * nblock * sizeof(block_iq2_xxs);+}++size_t quantize_iq2_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ GGML_ASSERT(n_per_row%QK_K == 0);+ int nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq2_xs);+ }+ return nrow * nblock * sizeof(block_iq2_xs);+}++//+// ============================================= 3-bit using D4 lattice+//++typedef struct {+ uint32_t * grid;+ int * map;+ uint16_t * neighbours;+} iq3_entry_t;++static iq3_entry_t iq3_data[1] = {+ {NULL, NULL, NULL},+};++static inline int iq3_data_index(int grid_size) {+ (void)grid_size;+ GGML_ASSERT(grid_size == 256);+ return 0;+}++static int iq3_compare_func(const void * left, const void * right) {+ const int * l = (const int *)left;+ const int * r = (const int *)right;+ return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;+}++void iq3xs_init_impl(int grid_size) {+ const int gindex = iq3_data_index(grid_size);+ if (iq3_data[gindex].grid) {+ return;+ }+ static const uint16_t kgrid_256[256] = {+ 0, 2, 4, 9, 11, 15, 16, 18, 25, 34, 59, 61, 65, 67, 72, 74,+ 81, 85, 88, 90, 97, 108, 120, 128, 130, 132, 137, 144, 146, 153, 155, 159,+ 169, 175, 189, 193, 199, 200, 202, 213, 248, 267, 287, 292, 303, 315, 317, 321,+ 327, 346, 362, 413, 436, 456, 460, 462, 483, 497, 513, 515, 520, 522, 529, 531,+ 536, 538, 540, 551, 552, 576, 578, 585, 592, 594, 641, 643, 648, 650, 657, 664,+ 698, 704, 706, 720, 729, 742, 758, 769, 773, 808, 848, 852, 870, 889, 901, 978,+ 992, 1024, 1026, 1033, 1035, 1040, 1042, 1046, 1049, 1058, 1089, 1091, 1093, 1096, 1098, 1105,+ 1112, 1139, 1143, 1144, 1152, 1154, 1161, 1167, 1168, 1170, 1183, 1184, 1197, 1217, 1224, 1228,+ 1272, 1276, 1309, 1323, 1347, 1367, 1377, 1404, 1473, 1475, 1486, 1509, 1537, 1544, 1546, 1553,+ 1555, 1576, 1589, 1594, 1600, 1602, 1616, 1625, 1636, 1638, 1665, 1667, 1672, 1685, 1706, 1722,+ 1737, 1755, 1816, 1831, 1850, 1856, 1862, 1874, 1901, 1932, 1950, 1971, 2011, 2032, 2052, 2063,+ 2077, 2079, 2091, 2095, 2172, 2192, 2207, 2208, 2224, 2230, 2247, 2277, 2308, 2345, 2356, 2389,+ 2403, 2424, 2501, 2504, 2506, 2520, 2570, 2593, 2616, 2624, 2630, 2646, 2669, 2700, 2714, 2746,+ 2754, 2795, 2824, 2835, 2839, 2874, 2882, 2905, 2984, 3028, 3042, 3092, 3108, 3110, 3124, 3153,+ 3185, 3215, 3252, 3288, 3294, 3364, 3397, 3434, 3483, 3523, 3537, 3587, 3589, 3591, 3592, 3610,+ 3626, 3670, 3680, 3722, 3749, 3754, 3776, 3789, 3803, 3824, 3857, 3873, 3904, 3906, 3924, 3992,+ };+ const int kmap_size = 4096;+ const int nwant = 2;+ const uint16_t * kgrid = kgrid_256;+ uint32_t * kgrid_q3xs;+ int * kmap_q3xs;+ uint16_t * kneighbors_q3xs;++ printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);+ uint32_t * the_grid = (uint32_t *)malloc(grid_size*sizeof(uint32_t));+ for (int k = 0; k < grid_size; ++k) {+ int8_t * pos = (int8_t *)(the_grid + k);+ for (int i = 0; i < 4; ++i) {+ int l = (kgrid[k] >> 3*i) & 0x7;+ pos[i] = 2*l + 1;+ }+ }+ kgrid_q3xs = the_grid;+ iq3_data[gindex].grid = the_grid;+ kmap_q3xs = (int *)malloc(kmap_size*sizeof(int));+ iq3_data[gindex].map = kmap_q3xs;+ for (int i = 0; i < kmap_size; ++i) kmap_q3xs[i] = -1;+ uint32_t aux32;+ uint8_t * aux8 = (uint8_t *)&aux32;+ for (int i = 0; i < grid_size; ++i) {+ aux32 = kgrid_q3xs[i];+ uint16_t index = 0;+ for (int k=0; k<4; ++k) {+ uint16_t q = (aux8[k] - 1)/2;+ index |= (q << 3*k);+ }+ kmap_q3xs[index] = i;+ }+ int8_t pos[4];+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));+ int num_neighbors = 0, num_not_in_map = 0;+ for (int i = 0; i < kmap_size; ++i) {+ if (kmap_q3xs[i] >= 0) continue;+ ++num_not_in_map;+ for (int k = 0; k < 4; ++k) {+ int l = (i >> 3*k) & 0x7;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);+ int d2 = 0;+ for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);dist2[2*j+0] = d2;dist2[2*j+1] = j;}- qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);+ qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);int n = 0; int d2 = dist2[0];int nhave = 1;for (int j = 0; j < grid_size; ++j) {@@ -5701,26 +7268,26 @@ void iq2xs_init_impl(int grid_size) {num_neighbors += n;}printf("%s: %d neighbours in total\n", __func__, num_neighbors);- kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));- iq2_data[gindex].neighbours = kneighbors_q2xs;+ kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));+ iq3_data[gindex].neighbours = kneighbors_q3xs;int counter = 0;for (int i = 0; i < kmap_size; ++i) {- if (kmap_q2xs[i] >= 0) continue;- for (int k = 0; k < 8; ++k) {- int l = (i >> 2*k) & 0x3;+ if (kmap_q3xs[i] >= 0) continue;+ for (int k = 0; k < 4; ++k) {+ int l = (i >> 3*k) & 0x7;pos[k] = 2*l + 1;}for (int j = 0; j < grid_size; ++j) {- const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + j);int d2 = 0;- for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);dist2[2*j+0] = d2;dist2[2*j+1] = j;}- qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);- kmap_q2xs[i] = -(counter + 1);+ qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func);+ kmap_q3xs[i] = -(counter + 1);int d2 = dist2[0];- uint16_t * start = &kneighbors_q2xs[counter++];+ uint16_t * start = &kneighbors_q3xs[counter++];int n = 0, nhave = 1;for (int j = 0; j < grid_size; ++j) {if (dist2[2*j] > d2) {@@ -5728,7 +7295,7 @@ void iq2xs_init_impl(int grid_size) {d2 = dist2[2*j];++nhave;}- kneighbors_q2xs[counter++] = dist2[2*j+1];+ kneighbors_q3xs[counter++] = dist2[2*j+1];++n;}*start = n;@@ -5736,17 +7303,17 @@ void iq2xs_init_impl(int grid_size) {free(dist2);}-void iq2xs_free_impl(int grid_size) {- GGML_ASSERT(grid_size == 256 || grid_size == 512 || grid_size == 1024);- const int gindex = iq2_data_index(grid_size);- if (iq2_data[gindex].grid) {- free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL;- free(iq2_data[gindex].map); iq2_data[gindex].map = NULL;- free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL;+void iq3xs_free_impl(int grid_size) {+ GGML_ASSERT(grid_size == 256);+ const int gindex = iq3_data_index(grid_size);+ if (iq3_data[gindex].grid) {+ free(iq3_data[gindex].grid); iq3_data[gindex].grid = NULL;+ free(iq3_data[gindex].map); iq3_data[gindex].map = NULL;+ free(iq3_data[gindex].neighbours); iq3_data[gindex].neighbours = NULL;}}-static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid,+static int iq3_find_best_neighbour(const uint16_t * restrict neighbours, const uint32_t * restrict grid,const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L) {int num_neighbors = neighbours[0];GGML_ASSERT(num_neighbors > 0);@@ -5755,7 +7322,7 @@ static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const ufor (int j = 1; j <= num_neighbors; ++j) {const int8_t * pg = (const int8_t *)(grid + neighbours[j]);float d2 = 0;- for (int i = 0; i < 8; ++i) {+ for (int i = 0; i < 4; ++i) {float q = pg[i];float diff = scale*q - xval[i];d2 += weight[i]*diff*diff;@@ -5766,45 +7333,46 @@ static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const u}GGML_ASSERT(grid_index >= 0);const int8_t * pg = (const int8_t *)(grid + grid_index);- for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ for (int i = 0; i < 4; ++i) L[i] = (pg[i] - 1)/2;return grid_index;}-static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {+static void quantize_row_iq3_xxs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {- const int gindex = iq2_data_index(256);+ const int gindex = iq3_data_index(256);- const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;- const int * kmap_q2xs = iq2_data[gindex].map;- const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;+ const uint32_t * kgrid_q3xs = iq3_data[gindex].grid;+ const int * kmap_q3xs = iq3_data[gindex].map;+ const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours;- GGML_ASSERT(quant_weights && "missing quantization weights");- GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");- GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");- GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ //GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?");GGML_ASSERT(n%QK_K == 0);- const int kMaxQ = 3;+ const int kMaxQ = 8;const int nbl = n/256;- block_iq2_xxs * y = vy;+ block_iq3_xxs * y = vy;float scales[QK_K/32];float weight[32];float xval[32];int8_t L[32];int8_t Laux[32];- float waux[32];- bool is_on_grid[4];- bool is_on_grid_aux[4];- uint8_t block_signs[4];- uint32_t q2[2*(QK_K/32)];+ float waux[32];+ bool is_on_grid[8];+ bool is_on_grid_aux[8];+ uint8_t block_signs[8];+ uint8_t q3[3*(QK_K/8)];+ uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K/4);for (int ibl = 0; ibl < nbl; ++ibl) {y[ibl].d = GGML_FP32_TO_FP16(0.f);- memset(q2, 0, QK_K/4);+ memset(q3, 0, 3*QK_K/8);float max_scale = 0;@@ -5815,8 +7383,12 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrictfor (int ib = 0; ib < QK_K/32; ++ib) {const float * xb = xbl + 32*ib;- const float * qw = quant_weights + QK_K*ibl + 32*ib;- for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*ibl + 32*ib;+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ } else {+ for (int i = 0; i < 32; ++i) weight[i] = xb[i]*xb[i];+ }for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);for (int k = 0; k < 4; ++k) {int nflip = 0;@@ -5849,22 +7421,22 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict}float best = 0;float scale = max/(2*kMaxQ-1);- for (int is = -9; is <= 9; ++is) {- float id = (2*kMaxQ-1+is*0.1f)/max;+ for (int is = -15; is <= 15; ++is) {+ float id = (2*kMaxQ-1+is*0.2f)/max;float this_scale = 1/id;- for (int k = 0; k < 4; ++k) {- for (int i = 0; i < 8; ++i) {- int l = nearest_int(0.5f*(id*xval[8*k+i]-1));- Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ for (int k = 0; k < 8; ++k) {+ for (int i = 0; i < 4; ++i) {+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));+ Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l));}uint16_t u = 0;- for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);- int grid_index = kmap_q2xs[u];+ for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i);+ int grid_index = kmap_q3xs[u];is_on_grid_aux[k] = true;if (grid_index < 0) {is_on_grid_aux[k] = false;- const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k);}}float sumqx = 0, sumq2 = 0;@@ -5877,28 +7449,28 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrictif (sumq2 > 0 && sumqx*sumqx > best*sumq2) {scale = sumqx/sumq2; best = scale*sumqx;for (int i = 0; i < 32; ++i) L[i] = Laux[i];- for (int k = 0; k < 4; ++k) is_on_grid[k] = is_on_grid_aux[k];+ for (int k = 0; k < 8; ++k) is_on_grid[k] = is_on_grid_aux[k];}}int n_not_ongrid = 0;- for (int k = 0; k < 4; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ for (int k = 0; k < 8; ++k) if (!is_on_grid[k]) ++n_not_ongrid;if (n_not_ongrid > 0 && scale > 0) {float id = 1/scale;- for (int k = 0; k < 4; ++k) {+ for (int k = 0; k < 8; ++k) {if (is_on_grid[k]) continue;uint16_t u = 0;- for (int i = 0; i < 8; ++i) {- int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ for (int i = 0; i < 4; ++i) {+ int l = nearest_int(0.5f*(id*xval[4*k+i]-1));l = MAX(0, MIN(kMaxQ-1, l));- u |= (l << 2*i);+ u |= (l << 3*i);}- int grid_index = kmap_q2xs[u];+ int grid_index = kmap_q3xs[u];if (grid_index < 0) {- const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1;+ grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k);}- const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);- for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;+ const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index);+ for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2;}float sumqx = 0, sumq2 = 0;for (int i = 0; i < 32; ++i) {@@ -5915,26 +7487,26 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrictscale = -scale;for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;}- for (int k = 0; k < 4; ++k) {+ for (int k = 0; k < 8; ++k) {uint16_t u = 0;- for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);- int grid_index = kmap_q2xs[u];+ for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i);+ int grid_index = kmap_q3xs[u];if (grid_index < 0) {printf("Oops: found point %u not on grid:", u);- for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]);printf("\n");GGML_ASSERT(false);}- q2[2*ib+0] |= (grid_index << 8*k);- q2[2*ib+1] |= (block_signs[k] << 7*k);+ q3[8*ib+k] = grid_index;}+ scales_and_signs[ib] = block_signs[0] | (block_signs[1] << 7) | (block_signs[2] << 14) | (block_signs[3] << 21);GGML_ASSERT(scale >= 0);scales[ib] = scale;max_scale = MAX(max_scale, scale);}if (!max_scale) {- memset(y[ibl].qs, 0, QK_K/4);+ memset(y[ibl].qs, 0, 3*QK_K/8);continue;}@@ -5945,86 +7517,179 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrictfor (int ib = 0; ib < QK_K/32; ++ib) {int l = nearest_int(0.5f*(id*scales[ib]-1));l = MAX(0, MIN(15, l));- q2[2*ib+1] |= ((uint32_t)l << 28);- const float * xb = xbl + 32*ib;- const float * qw = quant_weights + QK_K*ibl + 32*ib;- for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);- const uint8_t * aux8 = (const uint8_t *)(q2 + 2*ib);- const float db = d * (1 + 2*l);- uint32_t u = 0;- for (int k = 0; k < 4; ++k) {- const int8_t * signs = keven_signs_q2xs + 8*((q2[2*ib+1] >> 7*k) & 127);- const float * xk = xb + 8*k;- const float * wk = weight + 8*k;- const uint8_t * grid = (const uint8_t *)(kgrid_q2xs + aux8[k]);- float best_mse = 0; int best_index = aux8[k];- for (int j = 0; j < 8; ++j) {- float diff = db * grid[j] * signs[j] - xk[j];- best_mse += wk[j] * diff * diff;+ scales_and_signs[ib] |= ((uint32_t)l << 28);+ if (false) {+ const float * xb = xbl + 32*ib;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*ibl + 32*ib;+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ } else {+ for (int i = 0; i < 32; ++i) weight[i] = xb[i]*xb[i];}- for (int idx = 0; idx < 256; ++idx) {- grid = (const uint8_t *)(kgrid_q2xs + idx);- float mse = 0;- for (int j = 0; j < 8; ++j) {+ const float db = 0.25f * d * (1 + 2*l);+ for (int k = 0; k < 8; ++k) {+ const int8_t * signs = keven_signs_q2xs + 8*((scales_and_signs[ib] >> 7*(k/2)) & 127) + 4*(k%2);+ const float * xk = xb + 4*k;+ const float * wk = weight + 4*k;+ //const uint8_t * grid = (const uint8_t *)(kgrid_q3xs + q3[8*ib+k]);+ const uint8_t * grid = (const uint8_t *)(iq3xxs_grid + q3[8*ib+k]);+ float best_mse = 0; int best_index = q3[8*ib+k];+ for (int j = 0; j < 4; ++j) {float diff = db * grid[j] * signs[j] - xk[j];- mse += wk[j] * diff * diff;+ best_mse += wk[j] * diff * diff;}- if (mse < best_mse) {- best_mse = mse; best_index = idx;+ for (int idx = 0; idx < 256; ++idx) {+ //grid = (const uint8_t *)(kgrid_q3xs + idx);+ grid = (const uint8_t *)(iq3xxs_grid + idx);+ float mse = 0;+ for (int j = 0; j < 4; ++j) {+ float diff = db * grid[j] * signs[j] - xk[j];+ mse += wk[j] * diff * diff;+ }+ if (mse < best_mse) {+ best_mse = mse; best_index = idx;+ }+ }+ q3[8*ib+k] = best_index;+ //grid = (const uint8_t *)(kgrid_q3xs + best_index);+ grid = (const uint8_t *)(iq3xxs_grid + best_index);+ for (int j = 0; j < 4; ++j) {+ float q = db * grid[j] * signs[j];+ sumqx += wk[j] * q * xk[j];+ sumq2 += wk[j] * q * q;}}- u |= (best_index << 8*k);- grid = (const uint8_t *)(kgrid_q2xs + best_index);- //grid = (const uint8_t *)(kgrid_q2xs + aux8[k]);- for (int j = 0; j < 8; ++j) {- float q = db * grid[j] * signs[j];- sumqx += wk[j] * q * xk[j];- sumq2 += wk[j] * q * q;- }+ if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(d*sumqx/sumq2);}- q2[2*ib] = u;- if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(d*sumqx/sumq2);}- memcpy(y[ibl].qs, q2, QK_K/4);+ memcpy(y[ibl].qs, q3, 3*QK_K/8);}}-static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {+size_t quantize_iq3_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ GGML_ASSERT(n_per_row%QK_K == 0);+ int nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_iq3_xxs_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq3_xxs);+ }+ return nrow * nblock * sizeof(block_iq3_xxs);+}++void quantize_row_iq3_xxs(const float * restrict x, void * restrict vy, int k) {+ assert(k % QK_K == 0);+ block_iq3_xxs * restrict y = vy;+ quantize_row_iq3_xxs_reference(x, y, k);+}++void quantize_row_iq3_xxs_reference(const float * restrict x, block_iq3_xxs * restrict y, int k) {+ assert(k % QK_K == 0);+ quantize_row_iq3_xxs_impl(x, y, k, NULL);+}++// =================================== 1.5 bpw ===================================================++static int iq1_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid,+ const float * restrict xval, const float * restrict weight, float * scale, int8_t * restrict L, int ngrid) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_score = 0;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = (pg[i] - 3)/2;+ float w = weight[i];+ sumqx += w*q*xval[i];+ sumq2 += w*q*q;+ }+ if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ *scale = sumqx/sumq2; best_score = *scale * sumqx;+ grid_index = neighbours[j];+ }+ }+ if (grid_index < 0) {+ for (int i = 0; i < ngrid; ++i) {+ const int8_t * grid_i = (const int8_t *)(grid + i);+ float sumqx = 0, sumq2 = 0;+ for (int j = 0; j < 8; ++j) {+ float w = weight[j];+ float q = (grid_i[j] - 3)/2;+ sumqx += w*q*xval[j];+ sumq2 += w*q*q;+ }+ if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ *scale = sumqx/sumq2; best_score = *scale*sumqx;+ grid_index = i;+ }+ }+ }+ if (grid_index < 0) {+ printf("Oops, did not find grid point\n");+ printf("Have %d neighbours\n", num_neighbors);+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = (pg[i] - 3)/2;+ float w = weight[i];+ sumqx += w*q*xval[i];+ sumq2 += w*q*q;+ }+ printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2);+ }+ }+ GGML_ASSERT(grid_index >= 0);+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+ *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result.+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}++static int iq1_sort_helper(const void * left, const void * right) {+ const float * l = left;+ const float * r = right;+ return *l < *r ? -1 : *l > *r ? 1 : 0;+}- const int gindex = iq2_data_index(512);+static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {++ const int gindex = iq2_data_index(GGML_TYPE_IQ1_S);const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;const int * kmap_q2xs = iq2_data[gindex].map;const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;GGML_ASSERT(quant_weights && "missing quantization weights");- GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");GGML_ASSERT(n%QK_K == 0);- const int kMaxQ = 3;-const int nbl = n/256;- block_iq2_xs * y = vy;+ block_iq1_s * y = vy;- float scales[QK_K/16];- float weight[16];- float xval[16];- int8_t L[16];- int8_t Laux[16];- float waux[16];- bool is_on_grid[2];- bool is_on_grid_aux[2];- uint8_t block_signs[2];- uint16_t q2[2*(QK_K/16)];+ float scales[QK_K/8];+ float weight[8];+ int8_t L[8];+ float sumx[9];+ float sumw[9];+ float pairs[16];+ int * idx = (int *)(pairs + 1);+ uint8_t hbit[QK_K/8];for (int ibl = 0; ibl < nbl; ++ibl) {y[ibl].d = GGML_FP32_TO_FP16(0.f);- memset(q2, 0, QK_K/4);- memset(y[ibl].scales, 0, QK_K/32);+ memset(y[ibl].qs, 0, QK_K/8);+ memset(y[ibl].scales, 0, QK_K/16);float max_scale = 0;@@ -6033,165 +7698,217 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vfor (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];float sigma2 = sumx2/QK_K;- for (int ib = 0; ib < QK_K/16; ++ib) {- const float * xb = xbl + 16*ib;- const float * qw = quant_weights + QK_K*ibl + 16*ib;- for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);- for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);- for (int k = 0; k < 2; ++k) {- int nflip = 0;- uint8_t s = 0;- for (int i = 0; i < 8; ++i) {- if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];- else {- xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);- }- }- if (nflip%2) {- int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];- for (int i = 1; i < 8; ++i) {- float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];- if (ax < min) {- min = ax; imin = i;- }- }- xval[8*k+imin] = -xval[8*k+imin];- s ^= (1 << imin);- }- block_signs[k] = s & 127;- }- float max = xval[0];- for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);+ for (int ib = 0; ib < QK_K/8; ++ib) {+ const float * xb = xbl + 8*ib;+ const float * qw = quant_weights + QK_K*ibl + 8*ib;+ for (int i = 0; i < 8; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ float max = fabsf(xb[0]);+ for (int i = 1; i < 8; ++i) max = MAX(max, fabsf(xb[i]));if (!max) {scales[ib] = 0;- memset(L, 0, 16);+ memset(L, 1, 8);continue;}- float best = 0;- float scale = max/(2*kMaxQ-1);- is_on_grid[0] = is_on_grid[1] = true;- for (int is = -9; is <= 9; ++is) {- float id = (2*kMaxQ-1+is*0.1f)/max;- float this_scale = 1/id;- for (int k = 0; k < 2; ++k) {- for (int i = 0; i < 8; ++i) {- int l = nearest_int(0.5f*(id*xval[8*k+i]-1));- Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));- }- uint16_t u = 0;- for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);- int grid_index = kmap_q2xs[u];- is_on_grid_aux[k] = true;- if (grid_index < 0) {- is_on_grid_aux[k] = false;- const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);- }- }- float sumqx = 0, sumq2 = 0;- for (int i = 0; i < 16; ++i) {- float w = weight[i];- float q = 2*Laux[i] + 1;- sumqx += w*xval[i]*q;- sumq2 += w*q*q;- }- if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {- scale = sumqx/sumq2; best = scale*sumqx;- for (int i = 0; i < 16; ++i) L[i] = Laux[i];- for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];+ // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.+ // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two+ // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights+ // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and+ // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale+ // for each possible and score for each split.+ for (int j = 0; j < 8; ++j) {+ pairs[2*j] = xb[j];+ idx[2*j] = j;+ }+ qsort(pairs, 8, 2*sizeof(float), iq1_sort_helper);+ {+ sumx[0] = sumw[0] = 0;+ for (int j = 0; j < 8; ++j) {+ int i = idx[2*j];+ sumx[j+1] = sumx[j] + weight[i]*xb[i];+ sumw[j+1] = sumw[j] + weight[i];}}- int n_not_ongrid = 0;- for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;- if (n_not_ongrid > 0 && scale > 0) {- float id = 1/scale;- for (int k = 0; k < 2; ++k) {- if (is_on_grid[k]) continue;- uint16_t u = 0;- for (int i = 0; i < 8; ++i) {- int l = nearest_int(0.5f*(id*xval[8*k+i]-1));- l = MAX(0, MIN(kMaxQ-1, l));- u |= (l << 2*i);- L[8*k + i] = l;- }- int grid_index = kmap_q2xs[u];- if (grid_index < 0) {- const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ float best_score = 0, scale = max;+ int besti1 = 0, besti2 = 0;+ for (int i1 = 0; i1 <= 8; ++i1) {+ for (int i2 = i1; i2 <= 8; ++i2) {+ float sumqx = -(sumx[i1] - sumx[0]) + (sumx[8] - sumx[i2]);+ float sumq2 = (sumw[i1] - sumw[0]) + (sumw[8] - sumw[i2]);+ if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ scale = sumqx/sumq2; best_score = scale*sumqx;+ besti1 = i1; besti2 = i2;}}- float sumqx = 0, sumq2 = 0;- for (int i = 0; i < 16; ++i) {- float w = weight[i];- float q = 2*L[i] + 1;- sumqx += w*xval[i]*q;- sumq2 += w*q*q;- }- if (sumq2 > 0) scale = sumqx/sumq2;}+ for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;+ for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;+ for (int j = besti2; j < 8; ++j) L[idx[2*j]] = 2;if (scale < 0) {+ for (int j = 0; j < 8; ++j) L[j] = 2 - L[j];scale = -scale;- for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127;}- for (int k = 0; k < 2; ++k) {- uint16_t u = 0;- for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);- int grid_index = kmap_q2xs[u];- if (grid_index < 0) {- printf("Oops: found point %u not on grid:", u);- for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);- printf("\n");- GGML_ASSERT(false);- }- q2[2*ib+k] = grid_index | (block_signs[k] << 9);+ // Now we check if the solution found above corresponds to a grid point and, if not, use a neighbouring+ // grid point that minimizes SSD.+ uint16_t u = 0;+ for (int j = 0; j < 8; ++j) u |= (L[j] << 2*j);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq1_find_best_neighbour(neighbours, kgrid_q2xs, xb, weight, &scale, L, NGRID_IQ2XXS);+ GGML_ASSERT(grid_index >= 0);}+ y[ibl].qs[ib] = grid_index & 255;+ hbit[ib] = grid_index >> 8;GGML_ASSERT(scale >= 0);scales[ib] = scale;max_scale = MAX(max_scale, scale);}if (!max_scale) {- memset(y[ibl].qs, 0, QK_K/4);+ memset(y[ibl].qs, 0, QK_K/8);continue;}- float d = max_scale/31;- y[ibl].d = GGML_FP32_TO_FP16(d);+ float d = max_scale/15;+ y[ibl].d = GGML_FP32_TO_FP16(d*1.085f); // 1.085f is another fudge factor. Don't ask me why it is needed.float id = 1/d;- for (int ib = 0; ib < QK_K/16; ++ib) {+ for (int ib = 0; ib < QK_K/8; ++ib) {int l = nearest_int(0.5f*(id*scales[ib]-1));- l = MAX(0, MIN(15, l));- if (ib%2 == 0) y[ibl].scales[ib/2] = l;- else y[ibl].scales[ib/2] |= (l << 4);+ l = MAX(0, MIN(7, l));+ if (hbit[ib]) l |= 8;+ y[ibl].scales[ib/2] |= (l << 4*(ib%2));}- memcpy(y[ibl].qs, q2, QK_K/4);-}}-size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+size_t quantize_iq1_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {(void)hist;GGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;char * qrow = (char *)dst;for (int row = 0; row < nrow; ++row) {- quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights);+ quantize_row_iq1_s_impl(src, qrow, n_per_row, quant_weights);src += n_per_row;- qrow += nblock*sizeof(block_iq2_xxs);+ qrow += nblock*sizeof(block_iq1_s);}- return nrow * nblock * sizeof(block_iq2_xxs);+ return nrow * nblock * sizeof(block_iq1_s);}-size_t quantize_iq2_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+// ============================ 4-bit non-linear quants++static inline int best_index_int8(int n, const int8_t * val, float x) {+ if (x <= val[0]) return 0;+ if (x >= val[n-1]) return n-1;+ int ml = 0, mu = n-1;+ while (mu-ml > 1) {+ int mav = (ml+mu)/2;+ if (x < val[mav]) mu = mav; else ml = mav;+ }+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;+}++static void quantize_row_iq4_nl_impl(const int block_size, const float * GGML_RESTRICT x,+ ggml_fp16_t * dh, uint8_t * q4,+ float * weight, uint8_t * L,+ const int8_t * values,+ const float * quant_weights) {++ const int ntry = 7;++ float sigma2 = 0;+ for (int j = 0; j < QK4_NL; ++j) sigma2 += x[j]*x[j];+ sigma2 *= 2.f/QK4_NL;++ const int nb = QK4_NL/block_size;++ memset(q4, 0, QK4_NL/2);+ for (int ib = 0; ib < nb; ++ib) {+ dh[ib] = GGML_FP32_TO_FP16(0.f);+ const float * xb = x + ib*block_size;+ if (quant_weights) {+ const float * qw = quant_weights + ib*block_size;+ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ } else {+ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j];+ }+ float amax = 0, max = 0;+ for (int j = 0; j < block_size; ++j) {+ float ax = fabsf(xb[j]);+ if (ax > amax) {+ amax = ax; max = xb[j];+ }+ }+ if (!amax) {+ continue;+ }+ float d = -max/values[0];+ float id = 1/d;+ float sumqx = 0, sumq2 = 0;+ for (int j = 0; j < block_size; ++j) {+ float al = id*xb[j];+ int l = best_index_int8(16, values, al);+ float q = values[l];+ float w = weight[j];+ sumqx += w*q*xb[j];+ sumq2 += w*q*q;+ }+ float best_id = id;+ d = sumqx/sumq2;+ float best = d*sumqx;+ for (int itry = -ntry; itry <= ntry; ++itry) {+ id = (itry + values[0])/max;+ sumqx = sumq2 = 0;+ for (int j = 0; j < block_size; ++j) {+ float al = id*xb[j];+ int l = best_index_int8(16, values, al);+ float q = values[l];+ float w = weight[j];+ sumqx += w*q*xb[j];+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ d = sumqx/sumq2; best = d * sumqx;+ best_id = id;+ }+ }+ dh[ib] = GGML_FP32_TO_FP16(d);+ for (int j = 0; j < block_size; ++j) {+ L[ib*block_size + j] = best_index_int8(16, values, best_id*xb[j]);+ }+ }+ for (int i = 0; i < QK4_NL/32; ++i) {+ for (int j = 0; j < 16; ++j) {+ q4[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4);+ }+ }+}++size_t quantize_iq4_nl(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {(void)hist;- GGML_ASSERT(n_per_row%QK_K == 0);- int nblock = n_per_row/QK_K;+ GGML_ASSERT(n_per_row%QK4_NL == 0);+ int nblock = n_per_row/QK4_NL;char * qrow = (char *)dst;+ uint8_t L[QK4_NL];+ float weight[32];for (int row = 0; row < nrow; ++row) {- quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights);+ block_iq4_nl * iq4 = (block_iq4_nl *)qrow;+ for (int ibl = 0; ibl < nblock; ++ibl) {+ const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL;+ quantize_row_iq4_nl_impl(32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, weight, L, kvalues_iq4nl, qw);+ }src += n_per_row;- qrow += nblock*sizeof(block_iq2_xs);+ qrow += nblock*sizeof(block_iq4_nl);}- return nrow * nblock * sizeof(block_iq2_xs);+ return nrow * nblock * sizeof(block_iq4_nl);+}++void quantize_row_iq4_nl(const float * restrict x, void * restrict vy, int k) {+ assert(k % QK4_NL == 0);+ block_iq4_nl * restrict y = vy;+ quantize_row_iq4_nl_reference(x, y, k);}+void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * restrict y, int k) {+ assert(k % QK4_NL == 0);+ quantize_iq4_nl(x, y, 1, k, NULL, NULL);+}diff --git a/llama.cpp/ggml-quants.h b/llama.cpp/ggml-quants.hindex 7d7cf917..113623b6 100644--- a/llama.cpp/ggml-quants.h+++ b/llama.cpp/ggml-quants.h@@ -166,7 +166,7 @@ typedef struct {static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");// (Almost) "true" 2-bit quantization.-// Due to the need to use blocks as per ggml dsign, it ends up using+// Due to the need to use blocks as per ggml design, it ends up using// 2.0625 bpw because of the 16-bit scale for each block of 256.typedef struct {ggml_fp16_t d;@@ -182,72 +182,113 @@ typedef struct {} block_iq2_xs;static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");+// (Almost) "true" 3-bit quantization.+// Due to the need to use blocks as per ggml design, it ends up using+// 3.0625 bpw because of the 16-bit scale for each block of 256.+typedef struct {+ ggml_fp16_t d;+ uint8_t qs[3*QK_K/8];+} block_iq3_xxs;+static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");++typedef struct {+ ggml_fp16_t d;+ uint8_t qs[QK_K/8];+ uint8_t scales[QK_K/16];+} block_iq1_s;+static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");++// Non-linear quants+#define QK4_NL 32+typedef struct {+ ggml_fp16_t d;+ uint8_t qs[QK4_NL/2];+} block_iq4_nl;+static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding");++#ifdef __cplusplus+extern "C" {+#endif+// Quantization-void quantize_row_q4_0_reference(const float * restrict x, block_q4_0 * restrict y, int k);-void quantize_row_q4_1_reference(const float * restrict x, block_q4_1 * restrict y, int k);-void quantize_row_q5_0_reference(const float * restrict x, block_q5_0 * restrict y, int k);-void quantize_row_q5_1_reference(const float * restrict x, block_q5_1 * restrict y, int k);-void quantize_row_q8_0_reference(const float * restrict x, block_q8_0 * restrict y, int k);-void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict y, int k);--void quantize_row_q2_K_reference(const float * restrict x, block_q2_K * restrict y, int k);-void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k);-void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k);-void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k);-void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k);-void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k);--void quantize_row_q4_0(const float * restrict x, void * restrict y, int k);-void quantize_row_q4_1(const float * restrict x, void * restrict y, int k);-void quantize_row_q5_0(const float * restrict x, void * restrict y, int k);-void quantize_row_q5_1(const float * restrict x, void * restrict y, int k);-void quantize_row_q8_0(const float * restrict x, void * restrict y, int k);-void quantize_row_q8_1(const float * restrict x, void * restrict y, int k);--void quantize_row_q2_K(const float * restrict x, void * restrict y, int k);-void quantize_row_q3_K(const float * restrict x, void * restrict y, int k);-void quantize_row_q4_K(const float * restrict x, void * restrict y, int k);-void quantize_row_q5_K(const float * restrict x, void * restrict y, int k);-void quantize_row_q6_K(const float * restrict x, void * restrict y, int k);-void quantize_row_q8_K(const float * restrict x, void * restrict y, int k);+void quantize_row_q4_0_reference(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int k);+void quantize_row_q4_1_reference(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int k);+void quantize_row_q5_0_reference(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int k);+void quantize_row_q5_1_reference(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int k);+void quantize_row_q8_0_reference(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int k);+void quantize_row_q8_1_reference(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int k);++void quantize_row_q2_K_reference(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int k);+void quantize_row_q3_K_reference(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int k);+void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int k);+void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int k);+void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int k);+void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int k);+void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k);+void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int k);++void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);++void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);// Dequantization-void dequantize_row_q4_0(const block_q4_0 * restrict x, float * restrict y, int k);-void dequantize_row_q4_1(const block_q4_1 * restrict x, float * restrict y, int k);-void dequantize_row_q5_0(const block_q5_0 * restrict x, float * restrict y, int k);-void dequantize_row_q5_1(const block_q5_1 * restrict x, float * restrict y, int k);-void dequantize_row_q8_0(const block_q8_0 * restrict x, float * restrict y, int k);-//void dequantize_row_q8_1(const block_q8_1 * restrict x, float * restrict y, int k);--void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int k);-void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int k);-void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int k);-void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int k);-void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k);-void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k);-void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k);-void dequantize_row_iq2_xs (const block_iq2_xs * restrict x, float * restrict y, int k);+void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+//void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);++void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);// Dot product-void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy);--void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);-void ggml_vec_dot_iq2_xs_q8_K (int n, float * restrict s, const void * restrict vx, const void * restrict vy);+void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);++void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);//// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")//size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_iq4_nl (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);@@ -258,5 +299,12 @@ size_t quantize_q4_1 (const float * src, void * dst, int nrows, int n_per_row,size_t quantize_q5_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);size_t quantize_q5_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-void iq2xs_init_impl(int grid_size);-void iq2xs_free_impl(int grid_size);+void iq2xs_init_impl(enum ggml_type type);+void iq2xs_free_impl(enum ggml_type type);+void iq3xs_init_impl(int grid_size);+void iq3xs_free_impl(int grid_size);++#ifdef __cplusplus+}+#endif+diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 053bb44e..003f7c1e 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -1,37 +1,39 @@// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi-asm(".ident\t\"\\n\\n\-llama.cpp (MIT License)\\n\-Copyright (c) 2023 Georgi Gerganov\\n\-Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\\n\-Copyright (c) 2023 Yuji Hirose\\n\-Copyright (c) 2022 Niels Lohmann <https://nlohmann.me>\\n\-Copyright (c) 2022 Bjoern Hoehrmann <[email protected]>\\n\-Copyright (c) 2017 Sean Barrett\\n\-\\n\-Permission is hereby granted, free of charge, to any person obtaining a copy\\n\-of this software and associated documentation files (the \\\"Software\\\"), to deal\\n\-in the Software without restriction, including without limitation the rights\\n\-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\\n\-copies of the Software, and to permit persons to whom the Software is\\n\-furnished to do so, subject to the following conditions:\\n\-\\n\-The above copyright notice and this permission notice shall be included in all\\n\-copies or substantial portions of the Software.\\n\-\\n\-THE SOFTWARE IS PROVIDED \\\"AS IS\\\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\\n\-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\\n\-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\\n\-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\\n\-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\\n\-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\\n\-SOFTWARE.\"");+__notice(ggml_notice, "\+llama.cpp (MIT License)\n\+Copyright (c) 2023 Georgi Gerganov\n\+Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\n\+Copyright (c) 2023 Yuji Hirose\n\+Copyright (c) 2022 Niels Lohmann <https://nlohmann.me>\n\+Copyright (c) 2022 Bjoern Hoehrmann <[email protected]>\n\+Copyright (c) 2017 Sean Barrett\n\+\n\+Permission is hereby granted, free of charge, to any person obtaining a copy\n\+of this software and associated documentation files (the \"Software\"), to deal\n\+in the Software without restriction, including without limitation the rights\n\+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n\+copies of the Software, and to permit persons to whom the Software is\n\+furnished to do so, subject to the following conditions:\n\+\n\+The above copyright notice and this permission notice shall be included in all\n\+copies or substantial portions of the Software.\n\+\n\+THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n\+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n\+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n\+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n\+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n\+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n\+SOFTWARE.");#include "ggml-impl.h"#include "ggml-quants.h"#include "ggml-metal.h"#include "ggml-cuda.h"+#include "llamafile/llamafile.h"+#include "llama.cpp/ggml-impl.h"#include "llamafile/log.h"#include <alloca.h>@@ -185,6 +187,7 @@ inline static void * ggml_aligned_malloc(size_t size) {break;}GGML_PRINT("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));+ GGML_ASSERT(false);return NULL;}return aligned_memory;@@ -197,6 +200,38 @@ inline static void * ggml_aligned_malloc(size_t size) {#endif#endif+inline static void * ggml_malloc(size_t size) {+ if (size == 0) {+ GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_malloc!\n");+ return NULL;+ }+ void * result = malloc(size);+ if (result == NULL) {+ GGML_PRINT("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));+ GGML_ASSERT(false);+ }+ return result;+}++// calloc+inline static void * ggml_calloc(size_t num, size_t size) {+ if (num == 0 || size == 0) {+ GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_calloc!\n");+ return NULL;+ }+ void * result = calloc(num, size);+ if (result == NULL) {+ GGML_PRINT("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));+ GGML_ASSERT(false);+ }+ return result;+}++#define GGML_MALLOC(size) ggml_malloc(size)+#define GGML_CALLOC(num, size) ggml_calloc(num, size)++#define GGML_FREE(ptr) free(ptr)+#define UNUSED GGML_UNUSED#define SWAP(x, y, T) do { T SWAP = x; x = y; y = SWAP; } while (0)@@ -331,8 +366,8 @@ int64_t ggml_cycles_per_ms(void) {static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);-static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y);-static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y);+static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {[GGML_TYPE_I8] = {@@ -360,6 +395,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.is_quantized = false,.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32,.vec_dot_type = GGML_TYPE_F32,+ .nrows = 1,},[GGML_TYPE_F16] = {.type_name = "f16",@@ -371,6 +407,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) ggml_fp32_to_fp16_row,.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16,.vec_dot_type = GGML_TYPE_F16,+ .nrows = 1,},[GGML_TYPE_Q4_0] = {.type_name = "q4_0",@@ -382,6 +419,11 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q4_0_reference,.vec_dot = ggml_vec_dot_q4_0_q8_0,.vec_dot_type = GGML_TYPE_Q8_0,+#if defined (__ARM_FEATURE_MATMUL_INT8)+ .nrows = 2,+#else+ .nrows = 1,+#endif},[GGML_TYPE_Q4_1] = {.type_name = "q4_1",@@ -393,6 +435,11 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q4_1_reference,.vec_dot = ggml_vec_dot_q4_1_q8_1,.vec_dot_type = GGML_TYPE_Q8_1,+#if defined (__ARM_FEATURE_MATMUL_INT8)+ .nrows = 2,+#else+ .nrows = 1,+#endif},[4] = { // GGML_TYPE_Q4_2.type_name = "DEPRECATED",@@ -404,6 +451,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = NULL,.vec_dot = NULL,.vec_dot_type = GGML_TYPE_COUNT,+ .nrows = 1,},[5] = { // GGML_TYPE_Q4_3.type_name = "DEPRECATED",@@ -415,6 +463,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = NULL,.vec_dot = NULL,.vec_dot_type = GGML_TYPE_COUNT,+ .nrows = 1,},[GGML_TYPE_Q5_0] = {.type_name = "q5_0",@@ -426,6 +475,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q5_0_reference,.vec_dot = ggml_vec_dot_q5_0_q8_0,.vec_dot_type = GGML_TYPE_Q8_0,+ .nrows = 1,},[GGML_TYPE_Q5_1] = {.type_name = "q5_1",@@ -437,6 +487,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q5_1_reference,.vec_dot = ggml_vec_dot_q5_1_q8_1,.vec_dot_type = GGML_TYPE_Q8_1,+ .nrows = 1,},[GGML_TYPE_Q8_0] = {.type_name = "q8_0",@@ -448,6 +499,11 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q8_0_reference,.vec_dot = ggml_vec_dot_q8_0_q8_0,.vec_dot_type = GGML_TYPE_Q8_0,+#if defined (__ARM_FEATURE_MATMUL_INT8)+ .nrows = 2,+#else+ .nrows = 1,+#endif},[GGML_TYPE_Q8_1] = {.type_name = "q8_1",@@ -457,6 +513,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float = quantize_row_q8_1,.from_float_reference = (ggml_from_float_t) quantize_row_q8_1_reference,.vec_dot_type = GGML_TYPE_Q8_1,+ .nrows = 1,},[GGML_TYPE_Q2_K] = {.type_name = "q2_K",@@ -468,6 +525,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q2_K_reference,.vec_dot = ggml_vec_dot_q2_K_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,},[GGML_TYPE_Q3_K] = {.type_name = "q3_K",@@ -479,6 +537,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q3_K_reference,.vec_dot = ggml_vec_dot_q3_K_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,},[GGML_TYPE_Q4_K] = {.type_name = "q4_K",@@ -490,6 +549,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q4_K_reference,.vec_dot = ggml_vec_dot_q4_K_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,},[GGML_TYPE_Q5_K] = {.type_name = "q5_K",@@ -501,6 +561,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q5_K_reference,.vec_dot = ggml_vec_dot_q5_K_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,},[GGML_TYPE_Q6_K] = {.type_name = "q6_K",@@ -512,6 +573,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = (ggml_from_float_t) quantize_row_q6_K_reference,.vec_dot = ggml_vec_dot_q6_K_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,},[GGML_TYPE_IQ2_XXS] = {.type_name = "iq2_xxs",@@ -523,6 +585,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = NULL,.vec_dot = ggml_vec_dot_iq2_xxs_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,},[GGML_TYPE_IQ2_XS] = {.type_name = "iq2_xs",@@ -534,6 +597,43 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.from_float_reference = NULL,.vec_dot = ggml_vec_dot_iq2_xs_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,+ },+ [GGML_TYPE_IQ3_XXS] = {+ .type_name = "iq3_xxs",+ .blck_size = QK_K,+ .type_size = sizeof(block_iq3_xxs),+ .is_quantized = true,+ .to_float = (ggml_to_float_t) dequantize_row_iq3_xxs,+ .from_float = quantize_row_iq3_xxs,+ .from_float_reference = (ggml_from_float_t)quantize_row_iq3_xxs_reference,+ .vec_dot = ggml_vec_dot_iq3_xxs_q8_K,+ .vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,+ },+ [GGML_TYPE_IQ1_S] = {+ .type_name = "iq1_s",+ .blck_size = QK_K,+ .type_size = sizeof(block_iq1_s),+ .is_quantized = true,+ .to_float = (ggml_to_float_t) dequantize_row_iq1_s,+ .from_float = NULL,+ .from_float_reference = NULL,+ .vec_dot = ggml_vec_dot_iq1_s_q8_K,+ .vec_dot_type = GGML_TYPE_Q8_K,+ .nrows = 1,+ },+ [GGML_TYPE_IQ4_NL] = {+ .type_name = "iq4_nl",+ .blck_size = QK4_NL,+ .type_size = sizeof(block_iq4_nl),+ .is_quantized = true,+ .to_float = (ggml_to_float_t) dequantize_row_iq4_nl,+ .from_float = quantize_row_iq4_nl,+ .from_float_reference = (ggml_from_float_t)quantize_row_iq4_nl_reference,+ .vec_dot = ggml_vec_dot_iq4_nl_q8_0,+ .vec_dot_type = GGML_TYPE_Q8_0,+ .nrows = 1,},[GGML_TYPE_Q8_K] = {.type_name = "q8_K",@@ -1013,7 +1113,7 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {x[i] = _mm_add_ps(x[i], x[offset+i]); \} \const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \- res = _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \}// TODO: is this optimal ?@@ -1106,66 +1206,12 @@ inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x)inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }-#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx2")-#pragma GCC target("fma")-static void ggml_vec_dot_f32_avx2(const int n, float * restrict s, const float * restrict x, const float * restrict y) {- float sumf = 0.0f;- const int np = (n & ~(32 - 1));- __m256 sum[(32/8)] = { _mm256_setzero_ps() };- __m256 ax[(32/8)];- __m256 ay[(32/8)];- for (int i = 0; i < np; i += 32) {- for (int j = 0; j < (32/8); j++) {- ax[j] = _mm256_loadu_ps(x + i + j*8);- ay[j] = _mm256_loadu_ps(y + i + j*8);- sum[j] = _mm256_fmadd_ps(ax[j], ay[j], sum[j]);- }- }- GGML_F32x8_REDUCE_AVX(sumf, sum);- for (int i = np; i < n; ++i) {- sumf += x[i]*y[i];- }- *s = sumf;-}-#pragma GCC pop_options-#endif /* __x86_64__ */--#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx")-static void ggml_vec_dot_f32_avx(const int n, float * restrict s, const float * restrict x, const float * restrict y) {- float sumf = 0.0f;- const int np = (n & ~(32 - 1));- __m256 sum[(32/8)] = { _mm256_setzero_ps() };- __m256 ax[(32/8)];- __m256 ay[(32/8)];- for (int i = 0; i < np; i += 32) {- for (int j = 0; j < (32/8); j++) {- ax[j] = _mm256_loadu_ps(x + i + j*8);- ay[j] = _mm256_loadu_ps(y + i + j*8);- sum[j] = _mm256_add_ps(_mm256_mul_ps(ax[j], ay[j]), sum[j]);- }- }- GGML_F32x8_REDUCE_AVX(sumf, sum);- for (int i = np; i < n; ++i) {- sumf += x[i]*y[i];- }- *s = sumf;-}-#pragma GCC pop_options-#endif /* __x86_64__ */--static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y) {--#ifdef __x86_64__- if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {- return ggml_vec_dot_f32_avx2(n, s, x, y);- } else if (X86_HAVE(AVX)) {- return ggml_vec_dot_f32_avx(n, s, x, y);- }-#endif+static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);#ifdef GGML_SIMDfloat sumf = 0.0f;@@ -1230,7 +1276,12 @@ static void ggml_vec_dot_f16_avx(const int n, float * restrict s, ggml_fp16_t *#pragma GCC pop_options#endif /* __x86_64__ */-static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y) {+static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);#ifdef __x86_64__if (X86_HAVE(AVX) && X86_HAVE(F16C) && X86_HAVE(FMA)) {@@ -1443,7 +1494,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {#endif}-inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, x, x); *s = sqrtf(*s); }+inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }inline static void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }@@ -1900,9 +1951,16 @@ struct ggml_numa_node {};struct ggml_numa_nodes {+ enum ggml_numa_strategy numa_strategy;struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES];uint32_t n_nodes;uint32_t total_cpus; // hardware threads on system+ uint32_t current_node; // node on which main process is execting+#if defined(__gnu_linux__)+ cpu_set_t cpuset; // cpuset from numactl+#else+ uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype+#endif};//@@ -1936,13 +1994,30 @@ inline static void ggml_critical_section_end(void) {atomic_fetch_sub(&g_state_barrier, 1);}-void ggml_numa_init(void) {+#if defined(__gnu_linux__)+static cpu_set_t ggml_get_numa_affinity(void) {+ cpu_set_t cpuset;+ pthread_t thread;+ thread = pthread_self();+ CPU_ZERO(&cpuset);+ pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);+ return cpuset;+}+#else+static uint32_t ggml_get_numa_affinity(void) {+ return 0; // no NUMA support+}+#endif++void ggml_numa_init(enum ggml_numa_strategy numa_flag) {if (g_state.numa.n_nodes > 0) {fprintf(stderr, "ggml_numa_init: NUMA already initialized\n");return;}+#if defined(__gnu_linux__)+if (!IsLinux()) {return;}@@ -1951,6 +2026,13 @@ void ggml_numa_init(void) {char path[256];int rv;+ // set numa scheme+ g_state.numa.numa_strategy = numa_flag;++ GGML_PRINT_DEBUG("numa strategy %u\n",g_state.numa.numa_strategy);++ g_state.numa.cpuset = ggml_get_numa_affinity();+// enumerate nodeswhile (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES) {rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u", g_state.numa.n_nodes);@@ -1969,11 +2051,26 @@ void ggml_numa_init(void) {GGML_PRINT_DEBUG("found %u numa nodes, %u CPUs\n", g_state.numa.n_nodes, g_state.numa.total_cpus);- if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1) {+ // figure out which node we're on+ unsigned current_cpu;+ int getcpu_ret = 0;+#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 28)+ getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node);+#elif defined(__COSMOPOLITAN__)+ // TODO(jart): use getcpu() with next upgrade+ current_cpu = sched_getcpu();+#else+ // old glibc doesn't have a wrapper for this call. Fall back on direct syscall+ getcpu_ret = syscall(SYS_getcpu,¤t_cpu,&g_state.numa.current_node);+#endif++ if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) {g_state.numa.n_nodes = 0;return;}+ GGML_PRINT_DEBUG("found our process on numa node %u, CPU %u\n", g_state.numa.current_node, current_cpu);+for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) {struct ggml_numa_node * node = &g_state.numa.nodes[n];GGML_PRINT_DEBUG("CPUs on node %u:", n);@@ -1999,6 +2096,10 @@ void ggml_numa_init(void) {fclose(fptr);}}+#else+ GGML_UNUSED(numa_flag);+ // TODO+#endif}bool ggml_is_numa(void) {@@ -2176,6 +2277,9 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {case GGML_FTYPE_MOSTLY_Q6_K: wtype = GGML_TYPE_Q6_K; break;case GGML_FTYPE_MOSTLY_IQ2_XXS: wtype = GGML_TYPE_IQ2_XXS; break;case GGML_FTYPE_MOSTLY_IQ2_XS: wtype = GGML_TYPE_IQ2_XS; break;+ case GGML_FTYPE_MOSTLY_IQ3_XXS: wtype = GGML_TYPE_IQ3_XXS; break;+ case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break;+ case GGML_FTYPE_MOSTLY_IQ4_NL: wtype = GGML_TYPE_IQ4_NL; break;case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break;case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break;}@@ -2328,6 +2432,10 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {ggml_init_cublas();#if defined(GGML_USE_CLBLAST)ggml_cl_init();+#elif defined(GGML_USE_VULKAN)+ ggml_vk_init_cpu_assist();+#elif defined(GGML_USE_SYCL)+ ggml_init_sycl();#endifggml_setup_op_has_task_pass();@@ -2452,7 +2560,8 @@ size_t ggml_get_max_tensor_size(const struct ggml_context * ctx) {size_t max_size = 0;for (struct ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor != NULL; tensor = ggml_get_next_tensor(ctx, tensor)) {- max_size = MAX(max_size, ggml_nbytes(tensor));+ size_t bytes = ggml_nbytes(tensor);+ max_size = MAX(max_size, bytes);}return max_size;@@ -2588,7 +2697,7 @@ static struct ggml_tensor * ggml_new_tensor_impl(/*.nb =*/ { 0, 0, 0, 0 },/*.op =*/ GGML_OP_NONE,/*.op_params =*/ { 0 },- /*.is_param =*/ false,+ /*.flags =*/ 0,/*.grad =*/ NULL,/*.src =*/ { NULL },/*.perf_runs =*/ 0,@@ -3123,7 +3232,7 @@ const char * ggml_get_name(const struct ggml_tensor * tensor) {}struct ggml_tensor * ggml_set_name(struct ggml_tensor * tensor, const char * name) {- strncpy(tensor->name, name, sizeof(tensor->name));+ strncpy(tensor->name, name, sizeof(tensor->name) - 1);tensor->name[sizeof(tensor->name) - 1] = '\0';return tensor;}@@ -4999,16 +5108,28 @@ static struct ggml_tensor * ggml_soft_max_impl(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * mask,+ struct ggml_tensor * pos,float scale,+ float max_bias,bool inplace) {GGML_ASSERT(ggml_is_contiguous(a));+if (mask) {GGML_ASSERT(ggml_is_contiguous(mask));- GGML_ASSERT(mask->ne[2] == 1);- GGML_ASSERT(mask->ne[3] == 1);+ GGML_ASSERT(ggml_is_matrix(mask));GGML_ASSERT(ggml_can_repeat_rows(mask, a));}+ if (pos) {+ GGML_ASSERT(ggml_is_vector(pos));+ GGML_ASSERT(pos->type == GGML_TYPE_F32);+ GGML_ASSERT(pos->ne[0] == a->ne[0]);+ }++ if (max_bias > 0.0f) {+ GGML_ASSERT(pos);+ }+bool is_node = false;if (a->grad) {@@ -5017,13 +5138,14 @@ static struct ggml_tensor * ggml_soft_max_impl(struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);- float params[] = { scale };+ float params[] = { scale, max_bias };ggml_set_op_params(result, params, sizeof(params));result->op = GGML_OP_SOFT_MAX;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;result->src[1] = mask;+ result->src[2] = pos;return result;}@@ -5031,21 +5153,23 @@ static struct ggml_tensor * ggml_soft_max_impl(struct ggml_tensor * ggml_soft_max(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_soft_max_impl(ctx, a, NULL, 1.0f, false);+ return ggml_soft_max_impl(ctx, a, NULL, NULL, 1.0f, 0.0f, false);}struct ggml_tensor * ggml_soft_max_inplace(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_soft_max_impl(ctx, a, NULL, 1.0f, true);+ return ggml_soft_max_impl(ctx, a, NULL, NULL, 1.0f, 0.0f, true);}struct ggml_tensor * ggml_soft_max_ext(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * mask,- float scale) {- return ggml_soft_max_impl(ctx, a, mask, scale, false);+ struct ggml_tensor * pos,+ float scale,+ float max_bias) {+ return ggml_soft_max_impl(ctx, a, mask, pos, scale, max_bias, false);}// ggml_soft_max_back@@ -5331,7 +5455,7 @@ GGML_API struct ggml_tensor * ggml_conv_1d(int s0,int p0,int d0) {- struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false); // [N, OL, IC * K]+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F16); // [N, OL, IC * K]struct ggml_tensor * result =ggml_mul_mat(ctx,@@ -5409,16 +5533,15 @@ struct ggml_tensor * ggml_conv_depthwise_2d(int p1,int d0,int d1) {+struct ggml_tensor * new_a = ggml_reshape_4d(ctx, a, a->ne[0], a->ne[1], 1, a->ne[2] * a->ne[3]);struct ggml_tensor * im2col = ggml_im2col(ctx, new_a,ggml_reshape_4d(ctx, b, b->ne[0], b->ne[1], 1, b->ne[2] * b->ne[3]),- s0, s1, p0, p1, d0, d1, true); // [N * IC, OH, OW, KH * KW]-- struct ggml_tensor * result =- ggml_mul_mat(ctx,- ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1), // [OC,1, KH, KW] => [1, OC, 1, KH * KW]- ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3])); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW]+ s0, s1, p0, p1, d0, d1, true, GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW]+ struct ggml_tensor * new_b = ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3]); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW]+ new_a = ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1); // [OC,1, KH, KW] => [1, OC, 1, KH * KW]+ struct ggml_tensor * result = ggml_mul_mat(ctx, new_a, new_b);result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], b->ne[2], b->ne[3]); // [N, OC, OH, OW]return result;@@ -5439,7 +5562,8 @@ struct ggml_tensor * ggml_im2col(int p1,int d0,int d1,- bool is_2D) {+ bool is_2D,+ enum ggml_type dst_type) {if(is_2D) {GGML_ASSERT(a->ne[2] == b->ne[2]);@@ -5463,7 +5587,7 @@ struct ggml_tensor * ggml_im2col(is_2D ? b->ne[3] : 1,};- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F16, 4, ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, dst_type, 4, ne);int32_t params[] = { s0, s1, p0, p1, d0, d1, (is_2D ? 1 : 0) };ggml_set_op_params(result, params, sizeof(params));@@ -5488,14 +5612,16 @@ struct ggml_tensor * ggml_conv_2d(int p1,int d0,int d1) {- struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true); // [N, OH, OW, IC * KH * KW]+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true, GGML_TYPE_F16); // [N, OH, OW, IC * KH * KW]struct ggml_tensor * result =ggml_mul_mat(ctx,ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]), // [N, OH, OW, IC * KH * KW] => [N*OH*OW, IC * KH * KW]ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1] * a->ne[2]), a->ne[3])); // [OC,IC, KH, KW] => [OC, IC * KH * KW]- result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], a->ne[3], im2col->ne[3]); // [N, OC, OH, OW]+ result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], im2col->ne[3], a->ne[3]); // [OC, N, OH, OW]+ result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 1, 3, 2)); // [N, OC, OH, OW]+return result;}@@ -5614,12 +5740,13 @@ struct ggml_tensor * ggml_pool_2d(is_node = true;}+ struct ggml_tensor * result;const int64_t ne[3] = {ggml_calc_pool_output_size(a->ne[0], k0, s0, p0),ggml_calc_pool_output_size(a->ne[1], k1, s1, p1),a->ne[2],};- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 3, ne);+ result = ggml_new_tensor(ctx, GGML_TYPE_F32, 3, ne);int32_t params[] = { op, k0, k1, s0, s1, p0, p1 };ggml_set_op_params(result, params, sizeof(params));@@ -5627,7 +5754,6 @@ struct ggml_tensor * ggml_pool_2d(result->op = GGML_OP_POOL_2D;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;-return result;}@@ -6490,7 +6616,7 @@ struct ggml_tensor * ggml_cross_entropy_loss_back(void ggml_set_param(struct ggml_context * ctx,struct ggml_tensor * tensor) {- tensor->is_param = true;+ tensor->flags |= GGML_TENSOR_FLAG_PARAM;GGML_ASSERT(tensor->grad == NULL);tensor->grad = ggml_dup_tensor(ctx, tensor);@@ -6501,8 +6627,10 @@ void ggml_set_param(static void ggml_compute_forward_dup_same_cont(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));GGML_ASSERT(src0->type == dst->type);@@ -6533,8 +6661,10 @@ static void ggml_compute_forward_dup_same_cont(}static void ggml_compute_forward_dup_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -6547,7 +6677,7 @@ static void ggml_compute_forward_dup_f16(const int nth = params->nth; // number of threadsif (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) {- ggml_compute_forward_dup_same_cont(params, src0, dst);+ ggml_compute_forward_dup_same_cont(params, dst);return;}@@ -6804,8 +6934,10 @@ static void ggml_compute_forward_dup_f16(static void ggml_compute_forward_dup_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -6818,7 +6950,7 @@ static void ggml_compute_forward_dup_f32(const int nth = params->nth; // number of threadsif (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) {- ggml_compute_forward_dup_same_cont(params, src0, dst);+ ggml_compute_forward_dup_same_cont(params, dst);return;}@@ -7054,8 +7186,10 @@ static void ggml_compute_forward_dup_f32(// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.static void ggml_compute_forward_dup_bytes(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));GGML_ASSERT(src0->type == dst->type);@@ -7064,7 +7198,7 @@ static void ggml_compute_forward_dup_bytes(}if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {- ggml_compute_forward_dup_same_cont(params, src0, dst);+ ggml_compute_forward_dup_same_cont(params, dst);return;}@@ -7203,21 +7337,23 @@ static void ggml_compute_forward_dup_bytes(static void ggml_compute_forward_dup(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+if (src0->type == dst->type) {- ggml_compute_forward_dup_bytes(params, src0, dst);+ ggml_compute_forward_dup_bytes(params, dst);return;}switch (src0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_dup_f16(params, src0, dst);+ ggml_compute_forward_dup_f16(params, dst);} break;case GGML_TYPE_F32:{- ggml_compute_forward_dup_f32(params, src0, dst);+ ggml_compute_forward_dup_f32(params, dst);} break;default:{@@ -7230,9 +7366,11 @@ static void ggml_compute_forward_dup(static void ggml_compute_forward_add_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -7318,9 +7456,11 @@ static void ggml_compute_forward_add_f32(static void ggml_compute_forward_add_f16_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -7395,9 +7535,11 @@ static void ggml_compute_forward_add_f16_f32(static void ggml_compute_forward_add_f16_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -7449,9 +7591,11 @@ static void ggml_compute_forward_add_f16_f16(static void ggml_compute_forward_add_q_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -7527,14 +7671,16 @@ static void ggml_compute_forward_add_q_f32(static void ggml_compute_forward_add(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+switch (src0->type) {case GGML_TYPE_F32:{if (src1->type == GGML_TYPE_F32) {- ggml_compute_forward_add_f32(params, src0, src1, dst);+ ggml_compute_forward_add_f32(params, dst);}else {GGML_ASSERT(false);@@ -7543,10 +7689,10 @@ static void ggml_compute_forward_add(case GGML_TYPE_F16:{if (src1->type == GGML_TYPE_F16) {- ggml_compute_forward_add_f16_f16(params, src0, src1, dst);+ ggml_compute_forward_add_f16_f16(params, dst);}else if (src1->type == GGML_TYPE_F32) {- ggml_compute_forward_add_f16_f32(params, src0, src1, dst);+ ggml_compute_forward_add_f16_f32(params, dst);}else {GGML_ASSERT(false);@@ -7564,8 +7710,11 @@ static void ggml_compute_forward_add(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:{- ggml_compute_forward_add_q_f32(params, src0, src1, dst);+ ggml_compute_forward_add_q_f32(params, dst);} break;default:{@@ -7578,9 +7727,11 @@ static void ggml_compute_forward_add(static void ggml_compute_forward_add1_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));@@ -7630,9 +7781,11 @@ static void ggml_compute_forward_add1_f32(static void ggml_compute_forward_add1_f16_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));@@ -7680,9 +7833,11 @@ static void ggml_compute_forward_add1_f16_f32(static void ggml_compute_forward_add1_f16_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));@@ -7730,9 +7885,11 @@ static void ggml_compute_forward_add1_f16_f16(static void ggml_compute_forward_add1_q_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));@@ -7797,21 +7954,23 @@ static void ggml_compute_forward_add1_q_f32(static void ggml_compute_forward_add1(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_add1_f32(params, src0, src1, dst);+ ggml_compute_forward_add1_f32(params, dst);} break;case GGML_TYPE_F16:{if (src1->type == GGML_TYPE_F16) {- ggml_compute_forward_add1_f16_f16(params, src0, src1, dst);+ ggml_compute_forward_add1_f16_f16(params, dst);}else if (src1->type == GGML_TYPE_F32) {- ggml_compute_forward_add1_f16_f32(params, src0, src1, dst);+ ggml_compute_forward_add1_f16_f32(params, dst);}else {GGML_ASSERT(false);@@ -7830,8 +7989,11 @@ static void ggml_compute_forward_add1(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:{- ggml_compute_forward_add1_q_f32(params, src0, src1, dst);+ ggml_compute_forward_add1_q_f32(params, dst);} break;default:{@@ -7844,9 +8006,11 @@ static void ggml_compute_forward_add1(static void ggml_compute_forward_acc_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));@@ -7926,14 +8090,14 @@ static void ggml_compute_forward_acc_f32(static void ggml_compute_forward_acc(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_acc_f32(params, src0, src1, dst);+ ggml_compute_forward_acc_f32(params, dst);} break;case GGML_TYPE_F16:case GGML_TYPE_Q4_0:@@ -7949,6 +8113,9 @@ static void ggml_compute_forward_acc(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:default:{GGML_ASSERT(false);@@ -7960,9 +8127,11 @@ static void ggml_compute_forward_acc(static void ggml_compute_forward_sub_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));@@ -8020,13 +8189,14 @@ static void ggml_compute_forward_sub_f32(static void ggml_compute_forward_sub(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_sub_f32(params, src0, src1, dst);+ ggml_compute_forward_sub_f32(params, dst);} break;default:{@@ -8039,9 +8209,11 @@ static void ggml_compute_forward_sub(static void ggml_compute_forward_mul_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -8050,7 +8222,7 @@ static void ggml_compute_forward_mul_f32(const int ith = params->ith;const int nth = params->nth;-#ifdef GGML_USE_CLBLAST+#if defined(GGML_USE_CLBLAST)if (src1->backend == GGML_BACKEND_GPU) {// TODO: OpenCL kernel support full broadcastGGML_ASSERT(ggml_can_repeat_rows(src1, src0));@@ -8122,15 +8294,17 @@ static void ggml_compute_forward_mul_f32(static void ggml_compute_forward_mul(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(src1->type == GGML_TYPE_F32 && "only f32 src1 supported for now");switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_mul_f32(params, src0, src1, dst);+ ggml_compute_forward_mul_f32(params, dst);} break;default:{@@ -8143,9 +8317,11 @@ static void ggml_compute_forward_mul(static void ggml_compute_forward_div_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -8216,13 +8392,14 @@ static void ggml_compute_forward_div_f32(static void ggml_compute_forward_div(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_div_f32(params, src0, src1, dst);+ ggml_compute_forward_div_f32(params, dst);} break;default:{@@ -8235,8 +8412,10 @@ static void ggml_compute_forward_div(static void ggml_compute_forward_sqr_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -8259,12 +8438,14 @@ static void ggml_compute_forward_sqr_f32(static void ggml_compute_forward_sqr(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_sqr_f32(params, src0, dst);+ ggml_compute_forward_sqr_f32(params, dst);} break;default:{@@ -8277,8 +8458,10 @@ static void ggml_compute_forward_sqr(static void ggml_compute_forward_sqrt_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -8301,12 +8484,14 @@ static void ggml_compute_forward_sqrt_f32(static void ggml_compute_forward_sqrt(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_sqrt_f32(params, src0, dst);+ ggml_compute_forward_sqrt_f32(params, dst);} break;default:{@@ -8319,8 +8504,10 @@ static void ggml_compute_forward_sqrt(static void ggml_compute_forward_log_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_are_same_shape(src0, dst));@@ -8343,12 +8530,14 @@ static void ggml_compute_forward_log_f32(static void ggml_compute_forward_log(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_log_f32(params, src0, dst);+ ggml_compute_forward_log_f32(params, dst);} break;default:{@@ -8361,8 +8550,10 @@ static void ggml_compute_forward_log(static void ggml_compute_forward_sum_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_is_scalar(dst));@@ -8394,8 +8585,10 @@ static void ggml_compute_forward_sum_f32(static void ggml_compute_forward_sum_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_is_scalar(dst));@@ -8426,16 +8619,18 @@ static void ggml_compute_forward_sum_f16(static void ggml_compute_forward_sum(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_sum_f32(params, src0, dst);+ ggml_compute_forward_sum_f32(params, dst);} break;case GGML_TYPE_F16:{- ggml_compute_forward_sum_f16(params, src0, dst);+ ggml_compute_forward_sum_f16(params, dst);} break;default:{@@ -8448,8 +8643,10 @@ static void ggml_compute_forward_sum(static void ggml_compute_forward_sum_rows_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -8481,12 +8678,14 @@ static void ggml_compute_forward_sum_rows_f32(static void ggml_compute_forward_sum_rows(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_sum_rows_f32(params, src0, dst);+ ggml_compute_forward_sum_rows_f32(params, dst);} break;default:{@@ -8499,8 +8698,10 @@ static void ggml_compute_forward_sum_rows(static void ggml_compute_forward_mean_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -8536,12 +8737,14 @@ static void ggml_compute_forward_mean_f32(static void ggml_compute_forward_mean(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_mean_f32(params, src0, dst);+ ggml_compute_forward_mean_f32(params, dst);} break;default:{@@ -8554,8 +8757,10 @@ static void ggml_compute_forward_mean(static void ggml_compute_forward_argmax_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -8582,12 +8787,14 @@ static void ggml_compute_forward_argmax_f32(static void ggml_compute_forward_argmax(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_argmax_f32(params, src0, dst);+ ggml_compute_forward_argmax_f32(params, dst);} break;default:{@@ -8600,8 +8807,10 @@ static void ggml_compute_forward_argmax(static void ggml_compute_forward_repeat_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_can_repeat(src0, dst));@@ -8643,8 +8852,10 @@ static void ggml_compute_forward_repeat_f32(static void ggml_compute_forward_repeat_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_can_repeat(src0, dst));@@ -8689,18 +8900,20 @@ static void ggml_compute_forward_repeat_f16(static void ggml_compute_forward_repeat(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F16:case GGML_TYPE_I16:{- ggml_compute_forward_repeat_f16(params, src0, dst);+ ggml_compute_forward_repeat_f16(params, dst);} break;case GGML_TYPE_F32:case GGML_TYPE_I32:{- ggml_compute_forward_repeat_f32(params, src0, dst);+ ggml_compute_forward_repeat_f32(params, dst);} break;default:{@@ -8713,8 +8926,10 @@ static void ggml_compute_forward_repeat(static void ggml_compute_forward_repeat_back_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_can_repeat(dst, src0));@@ -8770,12 +8985,14 @@ static void ggml_compute_forward_repeat_back_f32(static void ggml_compute_forward_repeat_back(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_repeat_back_f32(params, src0, dst);+ ggml_compute_forward_repeat_back_f32(params, dst);} break;default:{@@ -8788,10 +9005,11 @@ static void ggml_compute_forward_repeat_back(static void ggml_compute_forward_concat_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -8836,14 +9054,15 @@ static void ggml_compute_forward_concat_f32(static void ggml_compute_forward_concat(const struct ggml_compute_params* params,- const struct ggml_tensor* src0,- const struct ggml_tensor* src1,struct ggml_tensor* dst) {- switch (src0->type) {++ const struct ggml_tensor * src0 = dst->src[0];++ switch (src0->type) {case GGML_TYPE_F32:case GGML_TYPE_I32:{- ggml_compute_forward_concat_f32(params, src0, src1, dst);+ ggml_compute_forward_concat_f32(params, dst);} break;default:{@@ -8856,8 +9075,10 @@ static void ggml_compute_forward_concat(static void ggml_compute_forward_abs_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -8880,12 +9101,14 @@ static void ggml_compute_forward_abs_f32(static void ggml_compute_forward_abs(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_abs_f32(params, src0, dst);+ ggml_compute_forward_abs_f32(params, dst);} break;default:{@@ -8898,8 +9121,10 @@ static void ggml_compute_forward_abs(static void ggml_compute_forward_sgn_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -8922,12 +9147,14 @@ static void ggml_compute_forward_sgn_f32(static void ggml_compute_forward_sgn(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_sgn_f32(params, src0, dst);+ ggml_compute_forward_sgn_f32(params, dst);} break;default:{@@ -8940,8 +9167,10 @@ static void ggml_compute_forward_sgn(static void ggml_compute_forward_neg_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -8964,12 +9193,14 @@ static void ggml_compute_forward_neg_f32(static void ggml_compute_forward_neg(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_neg_f32(params, src0, dst);+ ggml_compute_forward_neg_f32(params, dst);} break;default:{@@ -8982,8 +9213,10 @@ static void ggml_compute_forward_neg(static void ggml_compute_forward_step_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -9006,12 +9239,14 @@ static void ggml_compute_forward_step_f32(static void ggml_compute_forward_step(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_step_f32(params, src0, dst);+ ggml_compute_forward_step_f32(params, dst);} break;default:{@@ -9024,8 +9259,10 @@ static void ggml_compute_forward_step(static void ggml_compute_forward_tanh_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -9048,12 +9285,14 @@ static void ggml_compute_forward_tanh_f32(static void ggml_compute_forward_tanh(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_tanh_f32(params, src0, dst);+ ggml_compute_forward_tanh_f32(params, dst);} break;default:{@@ -9066,8 +9305,10 @@ static void ggml_compute_forward_tanh(static void ggml_compute_forward_elu_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -9090,12 +9331,14 @@ static void ggml_compute_forward_elu_f32(static void ggml_compute_forward_elu(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_elu_f32(params, src0, dst);+ ggml_compute_forward_elu_f32(params, dst);} break;default:{@@ -9108,8 +9351,10 @@ static void ggml_compute_forward_elu(static void ggml_compute_forward_relu_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -9132,12 +9377,14 @@ static void ggml_compute_forward_relu_f32(static void ggml_compute_forward_relu(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_relu_f32(params, src0, dst);+ ggml_compute_forward_relu_f32(params, dst);} break;default:{@@ -9150,8 +9397,10 @@ static void ggml_compute_forward_relu(static void ggml_compute_forward_gelu_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));@@ -9191,12 +9440,14 @@ static void ggml_compute_forward_gelu_f32(static void ggml_compute_forward_gelu(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_gelu_f32(params, src0, dst);+ ggml_compute_forward_gelu_f32(params, dst);} break;default:{@@ -9209,8 +9460,10 @@ static void ggml_compute_forward_gelu(static void ggml_compute_forward_gelu_quick_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));@@ -9250,12 +9503,14 @@ static void ggml_compute_forward_gelu_quick_f32(static void ggml_compute_forward_gelu_quick(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_gelu_quick_f32(params, src0, dst);+ ggml_compute_forward_gelu_quick_f32(params, dst);} break;default:{@@ -9268,8 +9523,10 @@ static void ggml_compute_forward_gelu_quick(static void ggml_compute_forward_silu_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));@@ -9309,12 +9566,14 @@ static void ggml_compute_forward_silu_f32(static void ggml_compute_forward_silu(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_silu_f32(params, src0, dst);+ ggml_compute_forward_silu_f32(params, dst);} break;default:{@@ -9326,8 +9585,10 @@ static void ggml_compute_forward_silu(static void ggml_compute_forward_leaky_relu_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -9353,12 +9614,14 @@ static void ggml_compute_forward_leaky_relu_f32(static void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_leaky_relu_f32(params, src0, dst);+ ggml_compute_forward_leaky_relu_f32(params, dst);} break;default:{@@ -9371,9 +9634,11 @@ static void ggml_compute_forward_leaky_relu(static void ggml_compute_forward_silu_back_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * grad,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * grad = dst->src[1];+GGML_ASSERT(ggml_is_contiguous_except_dim_1(grad));GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));@@ -9416,13 +9681,14 @@ static void ggml_compute_forward_silu_back_f32(static void ggml_compute_forward_silu_back(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * grad,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_silu_back_f32(params, src0, grad, dst);+ ggml_compute_forward_silu_back_f32(params, dst);} break;default:{@@ -9434,8 +9700,10 @@ static void ggml_compute_forward_silu_back(static void ggml_compute_forward_hardswish_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -9457,12 +9725,14 @@ static void ggml_compute_forward_hardswish_f32(}static void ggml_compute_forward_hardswish(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_hardswish_f32(params, src0, dst);+ ggml_compute_forward_hardswish_f32(params, dst);} break;default:{@@ -9473,8 +9743,10 @@ static void ggml_compute_forward_hardswish(static void ggml_compute_forward_hardsigmoid_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));@@ -9497,12 +9769,14 @@ static void ggml_compute_forward_hardsigmoid_f32(static void ggml_compute_forward_hardsigmoid(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_hardsigmoid_f32(params, src0, dst);+ ggml_compute_forward_hardsigmoid_f32(params, dst);} break;default:{@@ -9516,8 +9790,10 @@ static void ggml_compute_forward_hardsigmoid(static void ggml_compute_forward_norm_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -9569,12 +9845,14 @@ static void ggml_compute_forward_norm_f32(static void ggml_compute_forward_norm(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_norm_f32(params, src0, dst);+ ggml_compute_forward_norm_f32(params, dst);} break;default:{@@ -9587,8 +9865,10 @@ static void ggml_compute_forward_norm(static void ggml_compute_forward_rms_norm_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -9637,12 +9917,14 @@ static void ggml_compute_forward_rms_norm_f32(static void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_rms_norm_f32(params, src0, dst);+ ggml_compute_forward_rms_norm_f32(params, dst);} break;default:{@@ -9653,9 +9935,11 @@ static void ggml_compute_forward_rms_norm(static void ggml_compute_forward_rms_norm_back_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -9810,13 +10094,14 @@ static void ggml_compute_forward_rms_norm_back_f32(static void ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_rms_norm_back_f32(params, src0, src1, dst);+ ggml_compute_forward_rms_norm_back_f32(params, dst);} break;default:{@@ -9829,8 +10114,10 @@ static void ggml_compute_forward_rms_norm_back(static void ggml_compute_forward_group_norm_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -9901,12 +10188,14 @@ static void ggml_compute_forward_group_norm_f32(static void ggml_compute_forward_group_norm(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_group_norm_f32(params, src0, dst);+ ggml_compute_forward_group_norm_f32(params, dst);} break;default:{@@ -9952,9 +10241,11 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) {static void ggml_compute_forward_mul_mat(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -9970,6 +10261,7 @@ static void ggml_compute_forward_mul_mat(ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot;enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type;ggml_from_float_t const from_float_to_vec_dot = type_traits[vec_dot_type].from_float;+ int64_t const vec_dot_num_rows = type_traits[type].nrows;GGML_ASSERT(ne0 == ne01);GGML_ASSERT(ne1 == ne11);@@ -10005,7 +10297,7 @@ static void ggml_compute_forward_mul_mat(#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)if (ggml_compute_forward_mul_mat_use_blas(dst)) {const int64_t ne_plane = ne01*ne00;- const int64_t desired_wsize = ne13*ne12*ne_plane*sizeof(float);+ const size_t desired_wsize = ne13*ne12*ne_plane*sizeof(float);UNUSED(desired_wsize);if (params->type == GGML_TASK_INIT) {@@ -10161,12 +10453,23 @@ static void ggml_compute_forward_mul_mat(const int64_t blck_0 = 16;const int64_t blck_1 = 16;+ // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols+ int64_t nrc = vec_dot_num_rows;+ // TODO: currently the mmla kernels support only even numbered rows/cols.+ // this check can be removed once they are extended to support odd numbered rows/cols too+ if ((nr0 % 2 != 0) || (ne11 % 2 != 0)) {+ nrc = 1;+ }++ const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;+// attempt to reduce false-sharing (does not seem to make a difference)- float tmp[16];+ // 16 * 2, accounting for mmla kernels+ float tmp[32];for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) {for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) {- for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ++ir1) {+ for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ir1 += nrc) {const int64_t i13 = (ir1/(ne12*ne1));const int64_t i12 = (ir1 - i13*ne12*ne1)/ne1;const int64_t i11 = (ir1 - i13*ne12*ne1 - i12*ne1);@@ -10189,17 +10492,19 @@ static void ggml_compute_forward_mul_mat((src1_cont || src1->type != vec_dot_type? (i11 + i12*ne11 + i13*ne12*ne11)*row_size: (i11*nb11 + i12*nb12 + i13*nb13));-float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3));//for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) {// vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);//}- for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) {- vec_dot(ne00, &tmp[ir0 - iir0], src0_row + ir0*nb01, src1_col);+ for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ir0 += nrc) {+ vec_dot(ne00, &tmp[ir0 - iir0], (nrc>1 ? 16 : 0), src0_row + ir0*nb01, (nrc>1 ? nb01 : 0), src1_col, (nrc>1 ? src1_col_stride : 0), nrc);+ }++ for (int cn = 0; cn < nrc; ++cn) {+ memcpy(&dst_col[iir0 + cn*nb1/nb0], tmp + (cn*16), (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float));}- memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float));}}}@@ -10209,10 +10514,11 @@ static void ggml_compute_forward_mul_mat(static void ggml_compute_forward_mul_mat_id(const struct ggml_compute_params * params,- const struct ggml_tensor * ids,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {+ const struct ggml_tensor * ids = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+const struct ggml_tensor * src0 = dst->src[2]; // only for GGML_TENSOR_BINARY_OP_LOCALSGGML_TENSOR_BINARY_OP_LOCALS@@ -10388,7 +10694,7 @@ static void ggml_compute_forward_mul_mat_id(//}for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) {- vec_dot(ne00, &tmp[ir0 - iir0], src0_row + ir0*nb01, src1_col);+ vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_row + ir0*nb01, 0, src1_col, 0, 1);}memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float));}@@ -10403,9 +10709,11 @@ static void ggml_compute_forward_mul_mat_id(static void ggml_compute_forward_out_prod_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+// int64_t t0 = ggml_perf_time_us();// UNUSED(t0);@@ -10595,9 +10903,11 @@ static void ggml_compute_forward_out_prod_f32(static void ggml_compute_forward_out_prod_q_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+// int64_t t0 = ggml_perf_time_us();// UNUSED(t0);@@ -10708,9 +11018,10 @@ static void ggml_compute_forward_out_prod_q_f32(static void ggml_compute_forward_out_prod(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:@@ -10724,17 +11035,20 @@ static void ggml_compute_forward_out_prod(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:{- ggml_compute_forward_out_prod_q_f32(params, src0, src1, dst);+ ggml_compute_forward_out_prod_q_f32(params, dst);} break;case GGML_TYPE_F16:{GGML_ASSERT(false); // todo- // ggml_compute_forward_out_prod_f16_f32(params, src0, src1, dst);+ // ggml_compute_forward_out_prod_f16_f32(params, dst);} break;case GGML_TYPE_F32:{- ggml_compute_forward_out_prod_f32(params, src0, src1, dst);+ ggml_compute_forward_out_prod_f32(params, dst);} break;default:{@@ -10747,8 +11061,10 @@ static void ggml_compute_forward_out_prod(static void ggml_compute_forward_scale_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_is_contiguous(src0));GGML_ASSERT(ggml_is_contiguous(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));@@ -10789,12 +11105,14 @@ static void ggml_compute_forward_scale_f32(static void ggml_compute_forward_scale(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_scale_f32(params, src0, dst);+ ggml_compute_forward_scale_f32(params, dst);} break;default:{@@ -10807,9 +11125,11 @@ static void ggml_compute_forward_scale(static void ggml_compute_forward_set_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));@@ -10880,14 +11200,14 @@ static void ggml_compute_forward_set_f32(static void ggml_compute_forward_set(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_set_f32(params, src0, src1, dst);+ ggml_compute_forward_set_f32(params, dst);} break;case GGML_TYPE_F16:case GGML_TYPE_Q4_0:@@ -10903,6 +11223,9 @@ static void ggml_compute_forward_set(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:default:{GGML_ASSERT(false);@@ -10914,29 +11237,25 @@ static void ggml_compute_forward_set(static void ggml_compute_forward_cpy(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {- ggml_compute_forward_dup(params, src0, dst);+ ggml_compute_forward_dup(params, dst);}// ggml_compute_forward_contstatic void ggml_compute_forward_cont(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {- ggml_compute_forward_dup(params, src0, dst);+ ggml_compute_forward_dup(params, dst);}// ggml_compute_forward_reshapestatic void ggml_compute_forward_reshape(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {// NOPUNUSED(params);- UNUSED(src0);UNUSED(dst);}@@ -10944,39 +11263,41 @@ static void ggml_compute_forward_reshape(static void ggml_compute_forward_view(const struct ggml_compute_params * params,- const struct ggml_tensor * src0) {+ const struct ggml_tensor * dst) {// NOPUNUSED(params);- UNUSED(src0);+ UNUSED(dst);}// ggml_compute_forward_permutestatic void ggml_compute_forward_permute(const struct ggml_compute_params * params,- const struct ggml_tensor * src0) {+ const struct ggml_tensor * dst) {// NOPUNUSED(params);- UNUSED(src0);+ UNUSED(dst);}// ggml_compute_forward_transposestatic void ggml_compute_forward_transpose(const struct ggml_compute_params * params,- const struct ggml_tensor * src0) {+ const struct ggml_tensor * dst) {// NOPUNUSED(params);- UNUSED(src0);+ UNUSED(dst);}// ggml_compute_forward_get_rowsstatic void ggml_compute_forward_get_rows_q(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -11012,9 +11333,11 @@ static void ggml_compute_forward_get_rows_q(static void ggml_compute_forward_get_rows_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -11047,9 +11370,11 @@ static void ggml_compute_forward_get_rows_f16(static void ggml_compute_forward_get_rows_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -11082,9 +11407,10 @@ static void ggml_compute_forward_get_rows_f32(static void ggml_compute_forward_get_rows(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:@@ -11099,17 +11425,20 @@ static void ggml_compute_forward_get_rows(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:{- ggml_compute_forward_get_rows_q(params, src0, src1, dst);+ ggml_compute_forward_get_rows_q(params, dst);} break;case GGML_TYPE_F16:{- ggml_compute_forward_get_rows_f16(params, src0, src1, dst);+ ggml_compute_forward_get_rows_f16(params, dst);} break;case GGML_TYPE_F32:case GGML_TYPE_I32:{- ggml_compute_forward_get_rows_f32(params, src0, src1, dst);+ ggml_compute_forward_get_rows_f32(params, dst);} break;default:{@@ -11140,9 +11469,11 @@ static void ggml_compute_forward_get_rows(static void ggml_compute_forward_get_rows_back_f32_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_is_contiguous(dst));@@ -11177,9 +11508,11 @@ static void ggml_compute_forward_get_rows_back_f32_f16(static void ggml_compute_forward_get_rows_back_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_is_contiguous(dst));@@ -11214,17 +11547,18 @@ static void ggml_compute_forward_get_rows_back_f32(static void ggml_compute_forward_get_rows_back(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_get_rows_back_f32_f16(params, src0, src1, dst);+ ggml_compute_forward_get_rows_back_f32_f16(params, dst);} break;case GGML_TYPE_F32:{- ggml_compute_forward_get_rows_back_f32(params, src0, src1, dst);+ ggml_compute_forward_get_rows_back_f32(params, dst);} break;default:{@@ -11255,8 +11589,10 @@ static void ggml_compute_forward_get_rows_back(static void ggml_compute_forward_diag_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -11295,12 +11631,14 @@ static void ggml_compute_forward_diag_f32(static void ggml_compute_forward_diag(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_diag_f32(params, src0, dst);+ ggml_compute_forward_diag_f32(params, dst);} break;default:{@@ -11313,10 +11651,11 @@ static void ggml_compute_forward_diag(static void ggml_compute_forward_diag_mask_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst,const float value) {+ const struct ggml_tensor * src0 = dst->src[0];+const int ith = params->ith;const int nth = params->nth;@@ -11366,12 +11705,14 @@ static void ggml_compute_forward_diag_mask_f32(static void ggml_compute_forward_diag_mask_inf(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_diag_mask_f32(params, src0, dst, -INFINITY);+ ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY);} break;default:{@@ -11382,12 +11723,14 @@ static void ggml_compute_forward_diag_mask_inf(static void ggml_compute_forward_diag_mask_zero(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_diag_mask_f32(params, src0, dst, 0);+ ggml_compute_forward_diag_mask_f32(params, dst, 0);} break;default:{@@ -11400,9 +11743,12 @@ static void ggml_compute_forward_diag_mask_zero(static void ggml_compute_forward_soft_max_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+ const struct ggml_tensor * src2 = dst->src[2];+assert(ggml_is_contiguous(dst));assert(ggml_are_same_shape(src0, dst));@@ -11410,16 +11756,29 @@ static void ggml_compute_forward_soft_max_f32(return;}- float scale = 1.0f;- memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));+ float scale = 1.0f;+ float max_bias = 0.0f;++ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));+ memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));// TODO: handle transposed/permuted matricesconst int ith = params->ith;const int nth = params->nth;+ GGML_TENSOR_UNARY_OP_LOCALS+const int64_t ne11 = src1 ? src1->ne[1] : 1;+ // TODO: is this supposed to be ceil instead of floor?+ // https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370+ const uint32_t n_head_kv = ne02;+ const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head_kv));++ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);+const int nc = src0->ne[0];const int nr = ggml_nrows(src0);@@ -11432,6 +11791,9 @@ static void ggml_compute_forward_soft_max_f32(float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;+ // when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching+ float * pos = src2 ? (float *) src2->data : src0->data;+for (int i1 = ir0; i1 < ir1; i1++) {float * sp = (float *)((char *) src0->data + i1*src0->nb[1]);float * dp = (float *)((char *) dst->data + i1*dst->nb[1]);@@ -11445,6 +11807,16 @@ static void ggml_compute_forward_soft_max_f32(ggml_vec_acc_f32(nc, wp, mp);}+ // ALiBi bias+ if (max_bias > 0.0f) {+ const uint32_t h = (i1/ne01)%ne02; // head+ const float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1);++ for (int i = 0; i < nc; i++) {+ wp[i] = wp[i] + slope*pos[i];+ }+ }+#ifndef NDEBUGfor (int i = 0; i < nc; ++i) {//printf("p[%d] = %f\n", i, p[i]);@@ -11487,13 +11859,14 @@ static void ggml_compute_forward_soft_max_f32(static void ggml_compute_forward_soft_max(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_soft_max_f32(params, src0, src1, dst);+ ggml_compute_forward_soft_max_f32(params, dst);} break;default:{@@ -11506,9 +11879,11 @@ static void ggml_compute_forward_soft_max(static void ggml_compute_forward_soft_max_back_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_is_contiguous(src0));GGML_ASSERT(ggml_is_contiguous(src1));GGML_ASSERT(ggml_is_contiguous(dst));@@ -11567,7 +11942,7 @@ static void ggml_compute_forward_soft_max_back_f32(// linear runtime, no additional memoryfloat dot_y_dy = 0;- ggml_vec_dot_f32 (nc, &dot_y_dy, y, dy);+ ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1);ggml_vec_cpy_f32 (nc, dx, dy);ggml_vec_acc1_f32(nc, dx, -dot_y_dy);ggml_vec_mul_f32 (nc, dx, dx, y);@@ -11583,13 +11958,14 @@ static void ggml_compute_forward_soft_max_back_f32(static void ggml_compute_forward_soft_max_back(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_soft_max_back_f32(params, src0, src1, dst);+ ggml_compute_forward_soft_max_back_f32(params, dst);} break;default:{@@ -11602,8 +11978,10 @@ static void ggml_compute_forward_soft_max_back(static void ggml_compute_forward_alibi_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -11637,22 +12015,20 @@ static void ggml_compute_forward_alibi_f32(const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);- for (int64_t i = 0; i < ne0; i++) {- for (int64_t j = 0; j < ne1; j++) {- for (int64_t k = 0; k < ne2_ne3; k++) {- float * const src = (float *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);- float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);-- // TODO: k*nb2 or k*nb3-- float m_k;+ for (int64_t k = 0; k < ne2_ne3; k++) {+ // TODO: k*nb2 or k*nb3+ float m_k;- if (k < n_heads_log2_floor) {- m_k = powf(m0, k + 1);- } else {- m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);- }+ if (k < n_heads_log2_floor) {+ m_k = powf(m0, k + 1);+ } else {+ m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);+ }+ for (int64_t i = 0; i < ne0; i++) {+ for (int64_t j = 0; j < ne1; j++) {+ float * const src = (float *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);+ float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);pdst[0] = i * m_k + src[0];}}@@ -11661,8 +12037,10 @@ static void ggml_compute_forward_alibi_f32(static void ggml_compute_forward_alibi_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -11697,21 +12075,20 @@ static void ggml_compute_forward_alibi_f16(const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);- for (int i = 0; i < ne0; i++) {- for (int j = 0; j < ne1; j++) {- for (int k = 0; k < ne2_ne3; k++) {- ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);- float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);-- // TODO: k*nb2 or k*nb3+ for (int k = 0; k < ne2_ne3; k++) {+ // TODO: k*nb2 or k*nb3+ float m_k;- float m_k;+ if (k < n_heads_log2_floor) {+ m_k = powf(m0, k + 1);+ } else {+ m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);+ }- if (k < n_heads_log2_floor) {- m_k = powf(m0, k + 1);- } else {- m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);- }+ for (int i = 0; i < ne0; i++) {+ for (int j = 0; j < ne1; j++) {+ ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);+ float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);// we return F32pdst[0] = i * m_k + GGML_FP16_TO_FP32(src[0]);@@ -11722,16 +12099,18 @@ static void ggml_compute_forward_alibi_f16(static void ggml_compute_forward_alibi(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_alibi_f16(params, src0, dst);+ ggml_compute_forward_alibi_f16(params, dst);} break;case GGML_TYPE_F32:{- ggml_compute_forward_alibi_f32(params, src0, dst);+ ggml_compute_forward_alibi_f32(params, dst);} break;case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:@@ -11746,6 +12125,9 @@ static void ggml_compute_forward_alibi(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:case GGML_TYPE_Q8_K:case GGML_TYPE_I8:case GGML_TYPE_I16:@@ -11761,8 +12143,10 @@ static void ggml_compute_forward_alibi(static void ggml_compute_forward_clamp_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -11801,12 +12185,14 @@ static void ggml_compute_forward_clamp_f32(static void ggml_compute_forward_clamp(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_clamp_f32(params, src0, dst);+ ggml_compute_forward_clamp_f32(params, dst);} break;case GGML_TYPE_F16:case GGML_TYPE_Q4_0:@@ -11822,6 +12208,9 @@ static void ggml_compute_forward_clamp(case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:case GGML_TYPE_Q8_K:case GGML_TYPE_I8:case GGML_TYPE_I16:@@ -11885,16 +12274,20 @@ GGML_CALL void ggml_rope_yarn_corr_dims(int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]) {// start and end correction dims- dims[0] = MAX(0, floorf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_fast, freq_base)));- dims[1] = MIN(n_dims - 1, ceilf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_slow, freq_base)));+ float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_fast, freq_base));+ float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_slow, freq_base));+ dims[0] = MAX(0, start);+ dims[1] = MIN(n_dims - 1, end);}static void ggml_compute_forward_rope_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst,const bool forward) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -12067,10 +12460,12 @@ static void ggml_compute_forward_rope_f32(static void ggml_compute_forward_rope_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst,const bool forward) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -12232,17 +12627,18 @@ static void ggml_compute_forward_rope_f16(static void ggml_compute_forward_rope(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_rope_f16(params, src0, src1, dst, true);+ ggml_compute_forward_rope_f16(params, dst, true);} break;case GGML_TYPE_F32:{- ggml_compute_forward_rope_f32(params, src0, src1, dst, true);+ ggml_compute_forward_rope_f32(params, dst, true);} break;default:{@@ -12255,17 +12651,18 @@ static void ggml_compute_forward_rope(static void ggml_compute_forward_rope_back(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_rope_f16(params, src0, src1, dst, false);+ ggml_compute_forward_rope_f16(params, dst, false);} break;case GGML_TYPE_F32:{- ggml_compute_forward_rope_f32(params, src0, src1, dst, false);+ ggml_compute_forward_rope_f32(params, dst, false);} break;default:{@@ -12278,9 +12675,11 @@ static void ggml_compute_forward_rope_back(static void ggml_compute_forward_conv_transpose_1d_f16_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -12364,9 +12763,9 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(const int i1n = i10*ne11;for (int i00 = 0; i00 < ne00; i00++) {float v = 0;- ggml_vec_dot_f16(ne02, &v,- (ggml_fp16_t *) wdata_src + i1n,- (ggml_fp16_t *) wdata_kernel + i00*ne02);+ ggml_vec_dot_f16(ne02, &v, 0,+ (ggml_fp16_t *) wdata_src + i1n, 0,+ (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1);dst_data[i10*s0 + i00] += v;}}@@ -12375,9 +12774,11 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(static void ggml_compute_forward_conv_transpose_1d_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT(src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -12461,44 +12862,135 @@ static void ggml_compute_forward_conv_transpose_1d_f32(const int i1n = i10*ne11;for (int i00 = 0; i00 < ne00; i00++) {float v = 0;- ggml_vec_dot_f32(ne02, &v,- wdata_src + i1n,- wdata_kernel + i00*ne02);+ ggml_vec_dot_f32(ne02, &v, 0,+ wdata_src + i1n, 0,+ wdata_kernel + i00*ne02, 0, 1);dst_data[i10*s0 + i00] += v;}}}}-static void ggml_compute_forward_conv_transpose_1d(- const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- struct ggml_tensor * dst) {- switch (src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_conv_transpose_1d_f16_f32(params, src0, src1, dst);- } break;- case GGML_TYPE_F32:- {- ggml_compute_forward_conv_transpose_1d_f32(params, src0, src1, dst);- } break;- default:- {- GGML_ASSERT(false);- } break;- }-}+static void ggml_compute_forward_conv_transpose_1d(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ switch (src0->type) {+ case GGML_TYPE_F16:+ {+ ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst);+ } break;+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_conv_transpose_1d_f32(params, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}++// src0: kernel [OC, IC, KH, KW]+// src1: image [N, IC, IH, IW]+// dst: result [N, OH, OW, IC*KH*KW]+static void ggml_compute_forward_im2col_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ int64_t t0 = ggml_perf_time_us();+ UNUSED(t0);++ GGML_TENSOR_BINARY_OP_LOCALS;++ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;++ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t N = is_2D ? ne13 : ne12;+ const int64_t IC = is_2D ? ne12 : ne11;+ const int64_t IH = is_2D ? ne11 : 1;+ const int64_t IW = ne10;++ const int64_t KH = is_2D ? ne01 : 1;+ const int64_t KW = ne00;++ const int64_t OH = is_2D ? ne2 : 1;+ const int64_t OW = ne1;++ int ofs0 = is_2D ? nb13 : nb12;+ int ofs1 = is_2D ? nb12 : nb11;++ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nb10 == sizeof(float));++ if (params->type == GGML_TASK_INIT) {+ return;+ }++ if (params->type == GGML_TASK_FINALIZE) {+ return;+ }++ // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]+ {+ float * const wdata = (float *) dst->data;++ for (int64_t in = 0; in < N; in++) {+ for (int64_t ioh = 0; ioh < OH; ioh++) { // 1+ for (int64_t iow = 0; iow < OW; iow++) {+ for (int64_t iic = ith; iic < IC; iic += nth) {++ // micro kernel+ float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]+ const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]++ for (int64_t ikh = 0; ikh < KH; ikh++) { // 1+ for (int64_t ikw = 0; ikw < KW; ikw++) {+ const int64_t iiw = iow*s0 + ikw*d0 - p0;+ const int64_t iih = ioh*s1 + ikh*d1 - p1;++ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;+ } else {+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);+ }+ }+ }+ }+ }+ }+ }+ }+}+// src0: kernel [OC, IC, KH, KW]// src1: image [N, IC, IH, IW]// dst: result [N, OH, OW, IC*KH*KW]static void ggml_compute_forward_im2col_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F16);@@ -12578,17 +13070,15 @@ static void ggml_compute_forward_im2col_f16(static void ggml_compute_forward_im2col(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {- switch (src0->type) {+ switch (dst->type) {case GGML_TYPE_F16:{- ggml_compute_forward_im2col_f16(params, src0, src1, dst);+ ggml_compute_forward_im2col_f16(params, dst);} break;case GGML_TYPE_F32:{- GGML_ASSERT(false);+ ggml_compute_forward_im2col_f32(params, dst);} break;default:{@@ -12602,9 +13092,11 @@ static void ggml_compute_forward_im2col(static void ggml_compute_forward_conv_transpose_2d(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -12692,9 +13184,9 @@ static void ggml_compute_forward_conv_transpose_2d(for (int i01 = 0; i01 < ne01; i01++) {for (int i00 = 0; i00 < ne00; i00++) {float v = 0;- ggml_vec_dot_f16(ne03, &v,- wdata_src + i1n,- wdata_kernel + i01*ne00*ne03 + i00*ne03);+ ggml_vec_dot_f16(ne03, &v, 0,+ wdata_src + i1n, 0,+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;}}@@ -12708,9 +13200,11 @@ static void ggml_compute_forward_conv_transpose_2d(static void ggml_compute_forward_pool_1d_sk_p0(const struct ggml_compute_params * params,const enum ggml_op_pool op,- const struct ggml_tensor * src,const int k,struct ggml_tensor * dst) {++ const struct ggml_tensor * src = dst->src[0];+assert(src->type == GGML_TYPE_F32);assert(params->ith == 0);@@ -12759,7 +13253,6 @@ static void ggml_compute_forward_pool_1d_sk_p0(static void ggml_compute_forward_pool_1d(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {const int32_t * opts = (const int32_t *)dst->op_params;@@ -12770,17 +13263,19 @@ static void ggml_compute_forward_pool_1d(GGML_ASSERT(p0 == 0); // padding not supportedGGML_ASSERT(k0 == s0); // only s = k supported- ggml_compute_forward_pool_1d_sk_p0(params, op, src0, k0, dst);+ ggml_compute_forward_pool_1d_sk_p0(params, op, k0, dst);}// ggml_compute_forward_pool_2dstatic void ggml_compute_forward_pool_2d(const struct ggml_compute_params * params,- const struct ggml_tensor * src,struct ggml_tensor * dst) {- assert(src->type == GGML_TYPE_F32);- assert(params->ith == 0);++ const struct ggml_tensor * src = dst->src[0];++ GGML_ASSERT(src->type == GGML_TYPE_F32);+ GGML_ASSERT(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;@@ -12851,9 +13346,10 @@ static void ggml_compute_forward_pool_2d(static void ggml_compute_forward_upscale_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -12890,12 +13386,14 @@ static void ggml_compute_forward_upscale_f32(static void ggml_compute_forward_upscale(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_upscale_f32(params, src0, dst);+ ggml_compute_forward_upscale_f32(params, dst);} break;default:{@@ -12908,9 +13406,10 @@ static void ggml_compute_forward_upscale(static void ggml_compute_forward_pad_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -12948,12 +13447,14 @@ static void ggml_compute_forward_pad_f32(static void ggml_compute_forward_pad(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_pad_f32(params, src0, dst);+ ggml_compute_forward_pad_f32(params, dst);} break;default:{@@ -12966,9 +13467,10 @@ static void ggml_compute_forward_pad(static void ggml_compute_forward_argsort_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -13008,13 +13510,14 @@ static void ggml_compute_forward_argsort_f32(static void ggml_compute_forward_argsort(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_argsort_f32(params, src0, dst);+ ggml_compute_forward_argsort_f32(params, dst);} break;default:{@@ -13027,11 +13530,13 @@ static void ggml_compute_forward_argsort(static void ggml_compute_forward_flash_attn_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * q,- const struct ggml_tensor * k,- const struct ggml_tensor * v,const bool masked,struct ggml_tensor * dst) {++ const struct ggml_tensor * q = dst->src[0];+ const struct ggml_tensor * k = dst->src[1];+ const struct ggml_tensor * v = dst->src[2];+int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -13123,9 +13628,9 @@ static void ggml_compute_forward_flash_attn_f32(const int i1 = ik1;ggml_vec_dot_f32(neq0,- S + i1,- (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),- (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));+ S + i1, 0,+ (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,+ (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);}// scale@@ -13208,20 +13713,22 @@ static void ggml_compute_forward_flash_attn_f32(const int iv3 = iq3;ggml_vec_dot_f32(masked_begin,- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),- (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),- S);+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,+ (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,+ S, 0, 1);}}}static void ggml_compute_forward_flash_attn_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * q,- const struct ggml_tensor * k,- const struct ggml_tensor * v,const bool masked,struct ggml_tensor * dst) {++ const struct ggml_tensor * q = dst->src[0];+ const struct ggml_tensor * k = dst->src[1];+ const struct ggml_tensor * v = dst->src[2];+int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -13313,9 +13820,9 @@ static void ggml_compute_forward_flash_attn_f16(const int i1 = ik1;ggml_vec_dot_f16(neq0,- S + i1,- (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),- (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));+ S + i1, 0,+ (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,+ (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);}} else {for (int64_t ic = 0; ic < nek1; ic += GGML_VEC_DOT_UNROLL) {@@ -13417,9 +13924,9 @@ static void ggml_compute_forward_flash_attn_f16(const int iv3 = iq3;ggml_vec_dot_f16(nev0,- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),- (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),- S16);+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,+ (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,+ S16, 0, 1);}} else {for (int64_t ic = 0; ic < nev1; ic += GGML_VEC_DOT_UNROLL) {@@ -13443,19 +13950,19 @@ static void ggml_compute_forward_flash_attn_f16(static void ggml_compute_forward_flash_attn(const struct ggml_compute_params * params,- const struct ggml_tensor * q,- const struct ggml_tensor * k,- const struct ggml_tensor * v,const bool masked,struct ggml_tensor * dst) {++ const struct ggml_tensor * q = dst->src[0];+switch (q->type) {case GGML_TYPE_F16:{- ggml_compute_forward_flash_attn_f16(params, q, k, v, masked, dst);+ ggml_compute_forward_flash_attn_f16(params, masked, dst);} break;case GGML_TYPE_F32:{- ggml_compute_forward_flash_attn_f32(params, q, k, v, masked, dst);+ ggml_compute_forward_flash_attn_f32(params, masked, dst);} break;default:{@@ -13468,12 +13975,14 @@ static void ggml_compute_forward_flash_attn(static void ggml_compute_forward_flash_ff_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * a, // F16- const struct ggml_tensor * b0, // F16 fc_w- const struct ggml_tensor * b1, // F32 fc_b- const struct ggml_tensor * c0, // F16 proj_w- const struct ggml_tensor * c1, // F32 proj_bstruct ggml_tensor * dst) {++ const struct ggml_tensor * a = dst->src[0]; // F16+ const struct ggml_tensor * b0 = dst->src[1]; // F16 fc_w+ const struct ggml_tensor * b1 = dst->src[2]; // F32 fc_b+ const struct ggml_tensor * c0 = dst->src[3]; // F16 proj_w+ const struct ggml_tensor * c1 = dst->src[4]; // F32 proj_b+int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -13561,9 +14070,9 @@ static void ggml_compute_forward_flash_ff_f16(const int i1 = ib01;ggml_vec_dot_f16(nea0,- S + i1,- (ggml_fp16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)),- (ggml_fp16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3)));+ S + i1, 0,+ (ggml_fp16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)), 0,+ (ggml_fp16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3)), 0, 1);}ggml_vec_add_f32(neb01, S, S, (float *) b1->data);@@ -13586,9 +14095,9 @@ static void ggml_compute_forward_flash_ff_f16(for (int64_t ic = 0; ic < nec01; ++ic) {ggml_vec_dot_f16(neb01,- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),- (ggml_fp16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)),- S16);+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,+ (ggml_fp16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)), 0,+ S16, 0, 1);}ggml_vec_add_f32(nec01,@@ -13601,16 +14110,14 @@ static void ggml_compute_forward_flash_ff_f16(static void ggml_compute_forward_flash_ff(const struct ggml_compute_params * params,- const struct ggml_tensor * a,- const struct ggml_tensor * b0,- const struct ggml_tensor * b1,- const struct ggml_tensor * c0,- const struct ggml_tensor * c1,struct ggml_tensor * dst) {++ const struct ggml_tensor * b0 = dst->src[1];+switch (b0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_flash_ff_f16(params, a, b0, b1, c0, c1, dst);+ ggml_compute_forward_flash_ff_f16(params, dst);} break;case GGML_TYPE_F32:{@@ -13627,12 +14134,14 @@ static void ggml_compute_forward_flash_ff(static void ggml_compute_forward_flash_attn_back_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * q,- const struct ggml_tensor * k,- const struct ggml_tensor * v,- const struct ggml_tensor * d,const bool masked,struct ggml_tensor * dst) {++ const struct ggml_tensor * q = dst->src[0];+ const struct ggml_tensor * k = dst->src[1];+ const struct ggml_tensor * v = dst->src[2];+ const struct ggml_tensor * d = dst->src[3];+int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -13775,9 +14284,9 @@ static void ggml_compute_forward_flash_attn_back_f32(const int i1 = ik1;ggml_vec_dot_f32(neq0,- S + i1,- (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),- (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));+ S + i1, 0,+ (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,+ (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);}// scale@@ -13922,7 +14431,7 @@ static void ggml_compute_forward_flash_attn_back_f32(// S = SM * (S - dot(SM, S))float dot_SM_gradSM = 0;- ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, SM, S);+ ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1);ggml_vec_acc1_f32(M, S, -dot_SM_gradSM);ggml_vec_mul_f32 (masked_begin, S, S, SM);@@ -13980,16 +14489,15 @@ static void ggml_compute_forward_flash_attn_back_f32(static void ggml_compute_forward_flash_attn_back(const struct ggml_compute_params * params,- const struct ggml_tensor * q,- const struct ggml_tensor * k,- const struct ggml_tensor * v,- const struct ggml_tensor * d,const bool masked,struct ggml_tensor * dst) {++ const struct ggml_tensor * q = dst->src[0];+switch (q->type) {case GGML_TYPE_F32:{- ggml_compute_forward_flash_attn_back_f32(params, q, k, v, d, masked, dst);+ ggml_compute_forward_flash_attn_back_f32(params, masked, dst);} break;default:{@@ -14002,8 +14510,10 @@ static void ggml_compute_forward_flash_attn_back(static void ggml_compute_forward_win_part_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -14046,12 +14556,14 @@ static void ggml_compute_forward_win_part_f32(static void ggml_compute_forward_win_part(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_win_part_f32(params, src0, dst);+ ggml_compute_forward_win_part_f32(params, dst);} break;default:{@@ -14064,8 +14576,10 @@ static void ggml_compute_forward_win_part(static void ggml_compute_forward_win_unpart_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -14106,12 +14620,14 @@ static void ggml_compute_forward_win_unpart_f32(static void ggml_compute_forward_win_unpart(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_win_unpart_f32(params, src0, dst);+ ggml_compute_forward_win_unpart_f32(params, dst);} break;default:{@@ -14124,58 +14640,58 @@ static void ggml_compute_forward_win_unpart(static void ggml_compute_forward_unary(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {+const enum ggml_unary_op op = ggml_get_unary_op(dst);switch (op) {case GGML_UNARY_OP_ABS:{- ggml_compute_forward_abs(params, src0, dst);+ ggml_compute_forward_abs(params, dst);} break;case GGML_UNARY_OP_SGN:{- ggml_compute_forward_sgn(params, src0, dst);+ ggml_compute_forward_sgn(params, dst);} break;case GGML_UNARY_OP_NEG:{- ggml_compute_forward_neg(params, src0, dst);+ ggml_compute_forward_neg(params, dst);} break;case GGML_UNARY_OP_STEP:{- ggml_compute_forward_step(params, src0, dst);+ ggml_compute_forward_step(params, dst);} break;case GGML_UNARY_OP_TANH:{- ggml_compute_forward_tanh(params, src0, dst);+ ggml_compute_forward_tanh(params, dst);} break;case GGML_UNARY_OP_ELU:{- ggml_compute_forward_elu(params, src0, dst);+ ggml_compute_forward_elu(params, dst);} break;case GGML_UNARY_OP_RELU:{- ggml_compute_forward_relu(params, src0, dst);+ ggml_compute_forward_relu(params, dst);} break;case GGML_UNARY_OP_GELU:{- ggml_compute_forward_gelu(params, src0, dst);+ ggml_compute_forward_gelu(params, dst);} break;case GGML_UNARY_OP_GELU_QUICK:{- ggml_compute_forward_gelu_quick(params, src0, dst);+ ggml_compute_forward_gelu_quick(params, dst);} break;case GGML_UNARY_OP_SILU:{- ggml_compute_forward_silu(params, src0, dst);+ ggml_compute_forward_silu(params, dst);} break;case GGML_UNARY_OP_HARDSWISH:{- ggml_compute_forward_hardswish(params, src0, dst);+ ggml_compute_forward_hardswish(params, dst);} break;case GGML_UNARY_OP_HARDSIGMOID:{- ggml_compute_forward_hardsigmoid(params, src0, dst);+ ggml_compute_forward_hardsigmoid(params, dst);} break;default:{@@ -14188,8 +14704,10 @@ static void ggml_compute_forward_unary(static void ggml_compute_forward_get_rel_pos_f16(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -14215,12 +14733,14 @@ static void ggml_compute_forward_get_rel_pos_f16(static void ggml_compute_forward_get_rel_pos(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F16:{- ggml_compute_forward_get_rel_pos_f16(params, src0, dst);+ ggml_compute_forward_get_rel_pos_f16(params, dst);} break;default:{@@ -14233,11 +14753,12 @@ static void ggml_compute_forward_get_rel_pos(static void ggml_compute_forward_add_rel_pos_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * src2,struct ggml_tensor * dst) {+ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+ const struct ggml_tensor * src2 = dst->src[2];+const bool inplace = (bool) ((int32_t *) dst->op_params)[0];if (!inplace && params->type == GGML_TASK_INIT) {if (params->ith != 0) {@@ -14301,14 +14822,14 @@ static void ggml_compute_forward_add_rel_pos_f32(static void ggml_compute_forward_add_rel_pos(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * src2,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_add_rel_pos_f32(params, src0, src1, src2, dst);+ ggml_compute_forward_add_rel_pos_f32(params, dst);} break;default:{@@ -14321,9 +14842,11 @@ static void ggml_compute_forward_add_rel_pos(static void ggml_compute_forward_map_unary_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst,const ggml_unary_op_f32_t fun) {++ const struct ggml_tensor * src0 = dst->src[0];+GGML_ASSERT(ggml_are_same_shape(src0, dst));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -14345,13 +14868,15 @@ static void ggml_compute_forward_map_unary_f32(static void ggml_compute_forward_map_unary(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,struct ggml_tensor * dst,const ggml_unary_op_f32_t fun) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_map_unary_f32(params, src0, dst, fun);+ ggml_compute_forward_map_unary_f32(params, dst, fun);} break;default:{@@ -14364,10 +14889,12 @@ static void ggml_compute_forward_map_unary(static void ggml_compute_forward_map_binary_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst,const ggml_binary_op_f32_t fun) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+assert(params->ith == 0);assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));@@ -14392,14 +14919,15 @@ static void ggml_compute_forward_map_binary_f32(static void ggml_compute_forward_map_binary(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst,const ggml_binary_op_f32_t fun) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_map_binary_f32(params, src0, src1, dst, fun);+ ggml_compute_forward_map_binary_f32(params, dst, fun);} break;default:{@@ -14412,9 +14940,11 @@ static void ggml_compute_forward_map_binary(static void ggml_compute_forward_map_custom1_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * a,struct ggml_tensor * dst,const ggml_custom1_op_f32_t fun) {++ const struct ggml_tensor * a = dst->src[0];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -14428,10 +14958,12 @@ static void ggml_compute_forward_map_custom1_f32(static void ggml_compute_forward_map_custom2_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * a,- const struct ggml_tensor * b,struct ggml_tensor * dst,const ggml_custom2_op_f32_t fun) {++ const struct ggml_tensor * a = dst->src[0];+ const struct ggml_tensor * b = dst->src[1];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -14445,11 +14977,13 @@ static void ggml_compute_forward_map_custom2_f32(static void ggml_compute_forward_map_custom3_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * a,- const struct ggml_tensor * b,- const struct ggml_tensor * c,struct ggml_tensor * dst,const ggml_custom3_op_f32_t fun) {++ const struct ggml_tensor * a = dst->src[0];+ const struct ggml_tensor * b = dst->src[1];+ const struct ggml_tensor * c = dst->src[1];+assert(params->ith == 0);if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {@@ -14463,8 +14997,10 @@ static void ggml_compute_forward_map_custom3_f32(static void ggml_compute_forward_map_custom1(const struct ggml_compute_params * params,- const struct ggml_tensor * a,struct ggml_tensor * dst) {++ const struct ggml_tensor * a = dst->src[0];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -14478,9 +15014,11 @@ static void ggml_compute_forward_map_custom1(static void ggml_compute_forward_map_custom2(const struct ggml_compute_params * params,- const struct ggml_tensor * a,- const struct ggml_tensor * b,struct ggml_tensor * dst) {++ const struct ggml_tensor * a = dst->src[0];+ const struct ggml_tensor * b = dst->src[1];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -14494,10 +15032,12 @@ static void ggml_compute_forward_map_custom2(static void ggml_compute_forward_map_custom3(const struct ggml_compute_params * params,- const struct ggml_tensor * a,- const struct ggml_tensor * b,- const struct ggml_tensor * c,struct ggml_tensor * dst) {++ const struct ggml_tensor * a = dst->src[0];+ const struct ggml_tensor * b = dst->src[1];+ const struct ggml_tensor * c = dst->src[2];+if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}@@ -14511,9 +15051,11 @@ static void ggml_compute_forward_map_custom3(static void ggml_compute_forward_cross_entropy_loss_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+GGML_ASSERT(ggml_is_contiguous(src0));GGML_ASSERT(ggml_is_contiguous(src1));GGML_ASSERT(ggml_is_scalar(dst));@@ -14617,13 +15159,14 @@ static void ggml_compute_forward_cross_entropy_loss_f32(static void ggml_compute_forward_cross_entropy_loss(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_cross_entropy_loss_f32(params, src0, src1, dst);+ ggml_compute_forward_cross_entropy_loss_f32(params, dst);} break;default:{@@ -14636,10 +15179,12 @@ static void ggml_compute_forward_cross_entropy_loss(static void ggml_compute_forward_cross_entropy_loss_back_f32(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * opt0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+ const struct ggml_tensor * opt0 = dst->src[2];+GGML_ASSERT(ggml_is_contiguous(dst));GGML_ASSERT(ggml_is_contiguous(src0));GGML_ASSERT(ggml_is_contiguous(src1));@@ -14726,14 +15271,14 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(static void ggml_compute_forward_cross_entropy_loss_back(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * opt0,struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_cross_entropy_loss_back_f32(params, src0, src1, opt0, dst);+ ggml_compute_forward_cross_entropy_loss_back_f32(params, dst);} break;default:{@@ -14751,324 +15296,344 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggmreturn;}- if (ggml_cublas_loaded()) {+if (ggml_cublas_loaded()) {+/* #ifdef GGML_USE_CUBLAS */bool skip_cpu = ggml_cuda_compute_forward(params, tensor);if (skip_cpu) {return;}GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU);GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU);+/* #elif defined(GGML_USE_VULKAN) */+/* const bool skip_cpu = ggml_vk_compute_forward_cpu_assist(params, tensor); */+/* #ifdef GGML_VULKAN_CHECK_RESULTS */+/* if (skip_cpu) { */+/* ggml_vk_check_results_1_cpu_assist(params, tensor); */+/* } */+/* #endif */+/* if (skip_cpu) { */+/* return; */+/* } */+/* GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU); */+/* GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU); */+/* #endif // GGML_USE_CUBLAS */+}++#ifdef GGML_USE_SYCL+ bool skip_cpu = ggml_sycl_compute_forward(params, tensor);+ if (skip_cpu) {+ return;}-+#endif // GGML_USE_SYCLswitch (tensor->op) {case GGML_OP_DUP:{- ggml_compute_forward_dup(params, tensor->src[0], tensor);+ ggml_compute_forward_dup(params, tensor);} break;case GGML_OP_ADD:{- ggml_compute_forward_add(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_add(params, tensor);} break;case GGML_OP_ADD1:{- ggml_compute_forward_add1(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_add1(params, tensor);} break;case GGML_OP_ACC:{- ggml_compute_forward_acc(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_acc(params, tensor);} break;case GGML_OP_SUB:{- ggml_compute_forward_sub(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_sub(params, tensor);} break;case GGML_OP_MUL:{- ggml_compute_forward_mul(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_mul(params, tensor);} break;case GGML_OP_DIV:{- ggml_compute_forward_div(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_div(params, tensor);} break;case GGML_OP_SQR:{- ggml_compute_forward_sqr(params, tensor->src[0], tensor);+ ggml_compute_forward_sqr(params, tensor);} break;case GGML_OP_SQRT:{- ggml_compute_forward_sqrt(params, tensor->src[0], tensor);+ ggml_compute_forward_sqrt(params, tensor);} break;case GGML_OP_LOG:{- ggml_compute_forward_log(params, tensor->src[0], tensor);+ ggml_compute_forward_log(params, tensor);} break;case GGML_OP_SUM:{- ggml_compute_forward_sum(params, tensor->src[0], tensor);+ ggml_compute_forward_sum(params, tensor);} break;case GGML_OP_SUM_ROWS:{- ggml_compute_forward_sum_rows(params, tensor->src[0], tensor);+ ggml_compute_forward_sum_rows(params, tensor);} break;case GGML_OP_MEAN:{- ggml_compute_forward_mean(params, tensor->src[0], tensor);+ ggml_compute_forward_mean(params, tensor);} break;case GGML_OP_ARGMAX:{- ggml_compute_forward_argmax(params, tensor->src[0], tensor);+ ggml_compute_forward_argmax(params, tensor);} break;case GGML_OP_REPEAT:{- ggml_compute_forward_repeat(params, tensor->src[0], tensor);+ ggml_compute_forward_repeat(params, tensor);} break;case GGML_OP_REPEAT_BACK:{- ggml_compute_forward_repeat_back(params, tensor->src[0], tensor);+ ggml_compute_forward_repeat_back(params, tensor);} break;case GGML_OP_CONCAT:{- ggml_compute_forward_concat(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_concat(params, tensor);} break;case GGML_OP_SILU_BACK:{- ggml_compute_forward_silu_back(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_silu_back(params, tensor);} break;case GGML_OP_NORM:{- ggml_compute_forward_norm(params, tensor->src[0], tensor);+ ggml_compute_forward_norm(params, tensor);} break;case GGML_OP_RMS_NORM:{- ggml_compute_forward_rms_norm(params, tensor->src[0], tensor);+ ggml_compute_forward_rms_norm(params, tensor);} break;case GGML_OP_RMS_NORM_BACK:{- ggml_compute_forward_rms_norm_back(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_rms_norm_back(params, tensor);} break;case GGML_OP_GROUP_NORM:{- ggml_compute_forward_group_norm(params, tensor->src[0], tensor);+ ggml_compute_forward_group_norm(params, tensor);} break;case GGML_OP_MUL_MAT:{- ggml_compute_forward_mul_mat(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_mul_mat(params, tensor);} break;case GGML_OP_MUL_MAT_ID:{- ggml_compute_forward_mul_mat_id(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_mul_mat_id(params, tensor);} break;case GGML_OP_OUT_PROD:{- ggml_compute_forward_out_prod(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_out_prod(params, tensor);} break;case GGML_OP_SCALE:{- ggml_compute_forward_scale(params, tensor->src[0], tensor);+ ggml_compute_forward_scale(params, tensor);} break;case GGML_OP_SET:{- ggml_compute_forward_set(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_set(params, tensor);} break;case GGML_OP_CPY:{- ggml_compute_forward_cpy(params, tensor->src[0], tensor);+ ggml_compute_forward_cpy(params, tensor);} break;case GGML_OP_CONT:{- ggml_compute_forward_cont(params, tensor->src[0], tensor);+ ggml_compute_forward_cont(params, tensor);} break;case GGML_OP_RESHAPE:{- ggml_compute_forward_reshape(params, tensor->src[0], tensor);+ ggml_compute_forward_reshape(params, tensor);} break;case GGML_OP_VIEW:{- ggml_compute_forward_view(params, tensor->src[0]);+ ggml_compute_forward_view(params, tensor);} break;case GGML_OP_PERMUTE:{- ggml_compute_forward_permute(params, tensor->src[0]);+ ggml_compute_forward_permute(params, tensor);} break;case GGML_OP_TRANSPOSE:{- ggml_compute_forward_transpose(params, tensor->src[0]);+ ggml_compute_forward_transpose(params, tensor);} break;case GGML_OP_GET_ROWS:{- ggml_compute_forward_get_rows(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_get_rows(params, tensor);} break;case GGML_OP_GET_ROWS_BACK:{- ggml_compute_forward_get_rows_back(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_get_rows_back(params, tensor);} break;case GGML_OP_DIAG:{- ggml_compute_forward_diag(params, tensor->src[0], tensor);+ ggml_compute_forward_diag(params, tensor);} break;case GGML_OP_DIAG_MASK_INF:{- ggml_compute_forward_diag_mask_inf(params, tensor->src[0], tensor);+ ggml_compute_forward_diag_mask_inf(params, tensor);} break;case GGML_OP_DIAG_MASK_ZERO:{- ggml_compute_forward_diag_mask_zero(params, tensor->src[0], tensor);+ ggml_compute_forward_diag_mask_zero(params, tensor);} break;case GGML_OP_SOFT_MAX:{- ggml_compute_forward_soft_max(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_soft_max(params, tensor);} break;case GGML_OP_SOFT_MAX_BACK:{- ggml_compute_forward_soft_max_back(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_soft_max_back(params, tensor);} break;case GGML_OP_ROPE:{- ggml_compute_forward_rope(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_rope(params, tensor);} break;case GGML_OP_ROPE_BACK:{- ggml_compute_forward_rope_back(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_rope_back(params, tensor);} break;case GGML_OP_ALIBI:{- ggml_compute_forward_alibi(params, tensor->src[0], tensor);+ ggml_compute_forward_alibi(params, tensor);} break;case GGML_OP_CLAMP:{- ggml_compute_forward_clamp(params, tensor->src[0], tensor);+ ggml_compute_forward_clamp(params, tensor);} break;case GGML_OP_CONV_TRANSPOSE_1D:{- ggml_compute_forward_conv_transpose_1d(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_conv_transpose_1d(params, tensor);} break;case GGML_OP_IM2COL:{- ggml_compute_forward_im2col(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_im2col(params, tensor);} break;case GGML_OP_CONV_TRANSPOSE_2D:{- ggml_compute_forward_conv_transpose_2d(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_conv_transpose_2d(params, tensor);} break;case GGML_OP_POOL_1D:{- ggml_compute_forward_pool_1d(params, tensor->src[0], tensor);+ ggml_compute_forward_pool_1d(params, tensor);} break;case GGML_OP_POOL_2D:{- ggml_compute_forward_pool_2d(params, tensor->src[0], tensor);+ ggml_compute_forward_pool_2d(params, tensor);} break;case GGML_OP_UPSCALE:{- ggml_compute_forward_upscale(params, tensor->src[0], tensor);+ ggml_compute_forward_upscale(params, tensor);} break;case GGML_OP_PAD:{- ggml_compute_forward_pad(params, tensor->src[0], tensor);+ ggml_compute_forward_pad(params, tensor);} break;case GGML_OP_ARGSORT:{- ggml_compute_forward_argsort(params, tensor->src[0], tensor);+ ggml_compute_forward_argsort(params, tensor);} break;case GGML_OP_LEAKY_RELU:{- ggml_compute_forward_leaky_relu(params, tensor->src[0], tensor);+ ggml_compute_forward_leaky_relu(params, tensor);} break;case GGML_OP_FLASH_ATTN:{const int32_t t = ggml_get_op_params_i32(tensor, 0);GGML_ASSERT(t == 0 || t == 1);const bool masked = t != 0;- ggml_compute_forward_flash_attn(params, tensor->src[0], tensor->src[1], tensor->src[2], masked, tensor);+ ggml_compute_forward_flash_attn(params, masked, tensor);} break;case GGML_OP_FLASH_FF:{- ggml_compute_forward_flash_ff(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], tensor->src[4], tensor);+ ggml_compute_forward_flash_ff(params, tensor);} break;case GGML_OP_FLASH_ATTN_BACK:{int32_t t = ggml_get_op_params_i32(tensor, 0);GGML_ASSERT(t == 0 || t == 1);bool masked = t != 0;- ggml_compute_forward_flash_attn_back(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], masked, tensor);+ ggml_compute_forward_flash_attn_back(params, masked, tensor);} break;case GGML_OP_WIN_PART:{- ggml_compute_forward_win_part(params, tensor->src[0], tensor);+ ggml_compute_forward_win_part(params, tensor);} break;case GGML_OP_WIN_UNPART:{- ggml_compute_forward_win_unpart(params, tensor->src[0], tensor);+ ggml_compute_forward_win_unpart(params, tensor);} break;case GGML_OP_UNARY:{- ggml_compute_forward_unary(params, tensor->src[0], tensor);+ ggml_compute_forward_unary(params, tensor);} break;case GGML_OP_GET_REL_POS:{- ggml_compute_forward_get_rel_pos(params, tensor->src[0], tensor);+ ggml_compute_forward_get_rel_pos(params, tensor);} break;case GGML_OP_ADD_REL_POS:{- ggml_compute_forward_add_rel_pos(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor);+ ggml_compute_forward_add_rel_pos(params, tensor);} break;case GGML_OP_MAP_UNARY:{ggml_unary_op_f32_t fun;memcpy(&fun, tensor->op_params, sizeof(fun));- ggml_compute_forward_map_unary(params, tensor->src[0], tensor, fun);+ ggml_compute_forward_map_unary(params, tensor, fun);}break;case GGML_OP_MAP_BINARY:{ggml_binary_op_f32_t fun;memcpy(&fun, tensor->op_params, sizeof(fun));- ggml_compute_forward_map_binary(params, tensor->src[0], tensor->src[1], tensor, fun);+ ggml_compute_forward_map_binary(params, tensor, fun);}break;case GGML_OP_MAP_CUSTOM1_F32:{ggml_custom1_op_f32_t fun;memcpy(&fun, tensor->op_params, sizeof(fun));- ggml_compute_forward_map_custom1_f32(params, tensor->src[0], tensor, fun);+ ggml_compute_forward_map_custom1_f32(params, tensor, fun);}break;case GGML_OP_MAP_CUSTOM2_F32:{ggml_custom2_op_f32_t fun;memcpy(&fun, tensor->op_params, sizeof(fun));- ggml_compute_forward_map_custom2_f32(params, tensor->src[0], tensor->src[1], tensor, fun);+ ggml_compute_forward_map_custom2_f32(params, tensor, fun);}break;case GGML_OP_MAP_CUSTOM3_F32:{ggml_custom3_op_f32_t fun;memcpy(&fun, tensor->op_params, sizeof(fun));- ggml_compute_forward_map_custom3_f32(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor, fun);+ ggml_compute_forward_map_custom3_f32(params, tensor, fun);}break;case GGML_OP_MAP_CUSTOM1:{- ggml_compute_forward_map_custom1(params, tensor->src[0], tensor);+ ggml_compute_forward_map_custom1(params, tensor);}break;case GGML_OP_MAP_CUSTOM2:{- ggml_compute_forward_map_custom2(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_map_custom2(params, tensor);}break;case GGML_OP_MAP_CUSTOM3:{- ggml_compute_forward_map_custom3(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor);+ ggml_compute_forward_map_custom3(params, tensor);}break;case GGML_OP_CROSS_ENTROPY_LOSS:{- ggml_compute_forward_cross_entropy_loss(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_cross_entropy_loss(params, tensor);}break;case GGML_OP_CROSS_ENTROPY_LOSS_BACK:{- ggml_compute_forward_cross_entropy_loss_back(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor);+ ggml_compute_forward_cross_entropy_loss_back(params, tensor);}break;case GGML_OP_NONE:@@ -15162,13 +15727,13 @@ struct ggml_hash_set ggml_hash_set_new(size_t size) {size = ggml_hash_size(size);struct ggml_hash_set result;result.size = size;- result.keys = malloc(sizeof(struct ggml_tensor *) * size);+ result.keys = GGML_MALLOC(sizeof(struct ggml_tensor *) * size);memset(result.keys, 0, sizeof(struct ggml_tensor *) * size);return result;}static void ggml_hash_set_free(struct ggml_hash_set hash_set) {- free(hash_set.keys);+ GGML_FREE(hash_set.keys);}struct hash_map {@@ -15177,17 +15742,17 @@ struct hash_map {};static struct hash_map * ggml_new_hash_map(size_t size) {- struct hash_map * result = malloc(sizeof(struct hash_map));+ struct hash_map * result = GGML_MALLOC(sizeof(struct hash_map));result->set = ggml_hash_set_new(size);- result->vals = malloc(sizeof(struct ggml_tensor *) * result->set.size);+ result->vals = GGML_MALLOC(sizeof(struct ggml_tensor *) * result->set.size);memset(result->vals, 0, sizeof(struct ggml_tensor *) * result->set.size);return result;}static void ggml_hash_map_free(struct hash_map * map) {ggml_hash_set_free(map->set);- free(map->vals);- free(map);+ GGML_FREE(map->vals);+ GGML_FREE(map);}// gradient checkpointing@@ -15202,7 +15767,7 @@ static struct ggml_tensor * ggml_recompute_graph_node(return NULL;}- if (node->is_param) {+ if (node->flags & GGML_TENSOR_FLAG_PARAM) {return node;}@@ -15236,7 +15801,7 @@ static struct ggml_tensor * ggml_recompute_graph_node(clone->op = node->op;clone->grad = node->grad;- clone->is_param = node->is_param;+ clone->flags = node->flags;clone->extra = node->extra;for (int k = 0; k < GGML_MAX_DIMS; ++k) {clone->nb[k] = node->nb[k];@@ -16268,7 +16833,7 @@ void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph *for (int i = 0; i < gf->n_nodes; i++) {struct ggml_tensor * node = gf->nodes[i];- if (node->is_param) {+ if (node->flags & GGML_TENSOR_FLAG_PARAM) {GGML_PRINT_DEBUG("%s: found root node %p\n", __func__, (void *) node);ggml_build_forward_expand(gb, node->grad);}@@ -16472,26 +17037,47 @@ typedef pthread_t ggml_thread_t;#endif// Android's libc implementation "bionic" does not support setting affinity-static void set_numa_thread_affinity(int thread_n, int n_threads) {+#if defined(__gnu_linux__)+static void set_numa_thread_affinity(int thread_n) {if (!ggml_is_numa()) {return;}- // run thread on node_num thread_n / (threads per node)- const int node_num = thread_n / ((n_threads + g_state.numa.n_nodes - 1) / g_state.numa.n_nodes);- struct ggml_numa_node * node = &g_state.numa.nodes[node_num];+ int node_num;+ int rv;size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);+ switch(g_state.numa.numa_strategy) {+ case GGML_NUMA_STRATEGY_DISTRIBUTE:+ // run thread on node_num thread_n / (threads per node)+ node_num = thread_n % g_state.numa.n_nodes;+ break;+ case GGML_NUMA_STRATEGY_ISOLATE:+ // run thread on current_node+ node_num = g_state.numa.current_node;+ break;+ case GGML_NUMA_STRATEGY_NUMACTL:+ // use the cpuset that numactl gave us+ rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset);+ if (rv) {+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n",strerror(rv));+ }+ return;+ default:+ return;+ }++ struct ggml_numa_node * node = &g_state.numa.nodes[node_num];+cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);CPU_ZERO_S(setsize, cpus);for (size_t i = 0; i < node->n_cpus; ++i) {CPU_SET_S(node->cpus[i], setsize, cpus);}- int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);+ rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);if (rv) {- fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n",- strerror(rv));+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));}CPU_FREE(cpus);@@ -16512,12 +17098,17 @@ static void clear_numa_thread_affinity(void) {int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);if (rv) {- fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n",- strerror(rv));+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));}CPU_FREE(cpus);}+#else+// TODO: Windows etc.+// (the linux implementation may also work on BSD, someone should test)+static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); }+static void clear_numa_thread_affinity(void) {}+#endifstruct ggml_compute_state_shared {const struct ggml_cgraph * cgraph;@@ -16533,7 +17124,7 @@ struct ggml_compute_state_shared {atomic_int node_n; // active graph nodeatomic_int node_task; // active graph node task phase- bool (*abort_callback)(void * data); // abort ggml_graph_compute when true+ ggml_abort_callback abort_callback; // abort ggml_graph_compute when truevoid * abort_callback_data;};@@ -16815,7 +17406,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {const int n_threads = state->shared->n_threads;- set_numa_thread_affinity(state->ith, n_threads);+ set_numa_thread_affinity(state->ith);int node_n = -1;int task_phase = GGML_TASK_FINALIZE;@@ -16958,12 +17549,16 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threastruct ggml_cplan cplan;memset(&cplan, 0, sizeof(struct ggml_cplan));+ int max_tasks = 1;+// thread scheduling for the different operations + work buffer size estimationfor (int i = 0; i < cgraph->n_nodes; i++) {struct ggml_tensor * node = cgraph->nodes[i];const int n_tasks = ggml_get_n_tasks(node, n_threads);+ max_tasks = MAX(max_tasks, n_tasks);+size_t cur = 0;switch (node->op) {@@ -17130,7 +17725,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threawork_size += CACHE_LINE_SIZE*(n_threads - 1);}- cplan.n_threads = n_threads;+ cplan.n_threads = MIN(max_tasks, n_threads);cplan.work_size = work_size;cplan.work_data = NULL;@@ -17147,6 +17742,17 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {}}+#ifdef GGML_USE_VULKAN+ for (int i = 0; i < cgraph->n_nodes; i++) {+ ggml_vk_preallocate_buffers_graph_cpu_assist(cgraph->nodes[i]);+ }+ ggml_vk_preallocate_buffers_cpu_assist();++ for (int i = 0; i < cgraph->n_nodes; i++) {+ ggml_vk_build_graph_cpu_assist(cgraph->nodes[i], i == cgraph->n_nodes - 1);+ }+#endif+const int n_threads = cplan->n_threads;struct ggml_compute_state_shared state_shared = {@@ -17198,6 +17804,10 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {}}+#ifdef GGML_USE_VULKAN+ ggml_vk_graph_cleanup_cpu_assist();+#endif+// performance stats (graph){int64_t perf_cycles_cur = ggml_perf_cycles() - perf_start_cycles;@@ -17602,7 +18212,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *ptr += ggml_nbytes(tensor);- fprintf(stderr, "%s: loaded leaf %d: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));+ fprintf(stderr, "%s: loaded leaf %u: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));}}@@ -17705,7 +18315,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *result->nodes[i] = tensor;- fprintf(stderr, "%s: loaded node %d: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));+ fprintf(stderr, "%s: loaded node %u: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));}}}@@ -17727,7 +18337,7 @@ void ggml_graph_print(const struct ggml_cgraph * cgraph) {GGML_PRINT(" - %3d: [ %5" PRId64 ", %5" PRId64 ", %5" PRId64 "] %16s %s (%3d) cpu = %7.3f / %7.3f ms, wall = %7.3f / %7.3f ms\n",i,node->ne[0], node->ne[1], node->ne[2],- ggml_op_name(node->op), node->is_param ? "x" : node->grad ? "g" : " ", node->perf_runs,+ ggml_op_name(node->op), (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : node->grad ? "g" : " ", node->perf_runs,(double) node->perf_cycles / (double) ggml_cycles_per_ms(),(double) node->perf_cycles / (double) ggml_cycles_per_ms() / (double) node->perf_runs,(double) node->perf_time_us / 1000.0,@@ -17820,7 +18430,7 @@ void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraphcontinue;}- if (node->is_param) {+ if (node->flags & GGML_TENSOR_FLAG_PARAM) {snprintf(color, sizeof(color), "yellow");} else if (node->grad) {if (ggml_graph_find(gf, node)) {@@ -17994,7 +18604,7 @@ static enum ggml_opt_result ggml_opt_adam(int np = 0;int64_t nx = 0;for (int i = 0; i < gf->n_nodes; ++i) {- if (gf->nodes[i]->is_param) {+ if (gf->nodes[i]->flags & GGML_TENSOR_FLAG_PARAM) {GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op);GGML_ASSERT(np < GGML_MAX_PARAMS);@@ -18247,7 +18857,7 @@ static enum ggml_opt_result linesearch_backtracking(}// compute the initial gradient in the search direction- ggml_vec_dot_f32(nx, &dginit, g, d);+ ggml_vec_dot_f32(nx, &dginit, 0, g, 0, d, 0, 1);// make sure that d points to a descent directionif (0 < dginit) {@@ -18297,7 +18907,7 @@ static enum ggml_opt_result linesearch_backtracking(return count;}- ggml_vec_dot_f32(nx, &dg, g, d);+ ggml_vec_dot_f32(nx, &dg, 0, g, 0, d, 0, 1);// check the Wolfe conditionif (dg < params->lbfgs.wolfe * dginit) {@@ -18330,7 +18940,9 @@ static enum ggml_opt_result linesearch_backtracking((*step) *= width;}- GGML_UNREACHABLE();+ GGML_ASSERT(false && "line search failed");++ return GGML_LINESEARCH_FAIL;}static enum ggml_opt_result ggml_opt_lbfgs(@@ -18357,7 +18969,7 @@ static enum ggml_opt_result ggml_opt_lbfgs(int np = 0;int nx = 0;for (int i = 0; i < gf->n_nodes; ++i) {- if (gf->nodes[i]->is_param) {+ if (gf->nodes[i]->flags & GGML_TENSOR_FLAG_PARAM) {GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op);GGML_ASSERT(np < GGML_MAX_PARAMS);@@ -18558,8 +19170,8 @@ static enum ggml_opt_result ggml_opt_lbfgs(// ys = y^t \cdot s -> 1 / \rho.// yy = y^t \cdot y.//- ggml_vec_dot_f32(nx, &ys, &lm_y[end[0]*nx], &lm_s[end[0]*nx]);- ggml_vec_dot_f32(nx, &yy, &lm_y[end[0]*nx], &lm_y[end[0]*nx]);+ ggml_vec_dot_f32(nx, &ys, 0, &lm_y[end[0]*nx], 0, &lm_s[end[0]*nx], 0, 1);+ ggml_vec_dot_f32(nx, &yy, 0, &lm_y[end[0]*nx], 0, &lm_y[end[0]*nx], 0, 1);lm_ys[end[0]] = ys;@@ -18578,7 +19190,7 @@ static enum ggml_opt_result ggml_opt_lbfgs(for (int i = 0; i < bound; ++i) {j[0] = (j[0] + m - 1) % m;// \alpha_{j} = \rho_{j} s^{t}_{j} \cdot q_{k+1}- ggml_vec_dot_f32(nx, &lm_alpha[j[0]], &lm_s[j[0]*nx], d);+ ggml_vec_dot_f32(nx, &lm_alpha[j[0]], 0, &lm_s[j[0]*nx], 0, d, 0, 1);lm_alpha[j[0]] /= lm_ys[j[0]];// q_{i} = q_{i+1} - \alpha_{i} y_{i}ggml_vec_mad_f32(nx, d, &lm_y[j[0]*nx], -lm_alpha[j[0]]);@@ -18588,7 +19200,7 @@ static enum ggml_opt_result ggml_opt_lbfgs(for (int i = 0; i < bound; ++i) {// \beta_{j} = \rho_{j} y^t_{j} \cdot \gamma_{i}- ggml_vec_dot_f32(nx, &beta, &lm_y[j[0]*nx], d);+ ggml_vec_dot_f32(nx, &beta, 0, &lm_y[j[0]*nx], 0, d, 0, 1);beta /= lm_ys[j[0]];// \gamma_{i+1} = \gamma_{i} + (\alpha_{j} - \beta_{j}) s_{j}ggml_vec_mad_f32(nx, d, &lm_s[j[0]*nx], lm_alpha[j[0]] - beta);@@ -18598,7 +19210,9 @@ static enum ggml_opt_result ggml_opt_lbfgs(step[0] = 1.0;}- GGML_UNREACHABLE();+ GGML_ASSERT(false && "lbfgs failed");++ return GGML_OPT_DID_NOT_CONVERGE;}struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) {@@ -18832,12 +19446,24 @@ enum ggml_opt_result ggml_opt_resume_g(////////////////////////////////////////////////////////////////////////////////+void ggml_set_input(struct ggml_tensor * tensor) {+ tensor->flags |= GGML_TENSOR_FLAG_INPUT;+}++void ggml_set_output(struct ggml_tensor * tensor) {+ tensor->flags |= GGML_TENSOR_FLAG_OUTPUT;+}++////////////////////////////////////////////////////////////////////////////////+void ggml_quantize_init(enum ggml_type type) {ggml_critical_section_start();switch (type) {- case GGML_TYPE_IQ2_XXS: iq2xs_init_impl(256); break;- case GGML_TYPE_IQ2_XS: iq2xs_init_impl(512); break;+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ1_S: iq2xs_init_impl(type); break;+ case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break;default: // nothingbreak;}@@ -18848,8 +19474,10 @@ void ggml_quantize_init(enum ggml_type type) {void ggml_quantize_free(void) {ggml_critical_section_start();- iq2xs_free_impl(256);- iq2xs_free_impl(512);+ iq2xs_free_impl(GGML_TYPE_IQ2_XXS);+ iq2xs_free_impl(GGML_TYPE_IQ2_XS);+ iq2xs_free_impl(GGML_TYPE_IQ1_S);+ iq3xs_free_impl(256);ggml_critical_section_end();}@@ -18984,7 +19612,8 @@ size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t *bool ggml_quantize_requires_imatrix(enum ggml_type type) {returntype == GGML_TYPE_IQ2_XXS ||- type == GGML_TYPE_IQ2_XS;+ type == GGML_TYPE_IQ2_XS ||+ type == GGML_TYPE_IQ1_S;}size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start,@@ -19100,6 +19729,33 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, iresult = quantize_iq2_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);GGML_ASSERT(result == row_size * nrows);} break;+ case GGML_TYPE_IQ3_XXS:+ {+ GGML_ASSERT(start % QK_K == 0);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_iq3_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);+ } break;+ case GGML_TYPE_IQ1_S:+ {+ GGML_ASSERT(start % QK_K == 0);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_iq1_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);+ } break;+ case GGML_TYPE_IQ4_NL:+ {+ GGML_ASSERT(start % QK4_NL == 0);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_iq4_nl(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);+ } break;case GGML_TYPE_F16:{size_t elemsize = sizeof(ggml_fp16_t);@@ -19226,6 +19882,25 @@ struct gguf_context {void * data;};+static size_t gguf_type_size(enum gguf_type type) {+ GGML_ASSERT(0 <= type && type < GGUF_TYPE_COUNT);+ return GGUF_TYPE_SIZE[type];+}++static void gguf_tensor_info_sanitize(struct gguf_tensor_info * info) {+ GGML_ASSERT(info->n_dims <= GGML_MAX_DIMS);+ GGML_ASSERT(0 <= info->type && info->type < GGML_TYPE_COUNT);++ for (uint32_t i = 0; i < info->n_dims; ++i) {+ GGML_ASSERT(info->ne[i] > 0);+ }++ // prevent overflow for total number of elements+ GGML_ASSERT(INT64_MAX/info->ne[1] > info->ne[0]);+ GGML_ASSERT(INT64_MAX/info->ne[2] > info->ne[0]*info->ne[1]);+ GGML_ASSERT(INT64_MAX/info->ne[3] > info->ne[0]*info->ne[1]*info->ne[2]);+}+static bool gguf_fread_el(struct llamafile * file, void * dst, size_t size, size_t * offset) {const size_t n = llamafile_read(file, dst, size);*offset += n;@@ -19238,8 +19913,17 @@ static bool gguf_fread_str(struct llamafile * file, struct gguf_str * p, size_tbool ok = true;- ok = ok && gguf_fread_el(file, &p->n, sizeof(p->n), offset); p->data = calloc(p->n + 1, 1);- ok = ok && gguf_fread_el(file, p->data, p->n, offset);+ ok = ok && gguf_fread_el(file, &p->n, sizeof(p->n), offset);++ // early exit if string length is invalid, prevents from integer overflow+ if (p->n == SIZE_MAX) {+ fprintf(stderr, "%s: invalid string length (%" PRIu64 ")\n", __func__, p->n);+ return false;+ }++ p->data = GGML_CALLOC(p->n + 1, 1);++ ok = ok && gguf_fread_el(file, p->data, p->n, offset);return ok;}@@ -19277,11 +19961,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_ifor (uint32_t i = 0; i < sizeof(magic); i++) {if (magic[i] != GGUF_MAGIC[i]) {- fprintf(stderr, "%s: invalid magic characters: %c%c%c%c.\n", __func__,- isgraph(magic[0]) ? magic[0] : '?',- isgraph(magic[1]) ? magic[1] : '?',- isgraph(magic[2]) ? magic[2] : '?',- isgraph(magic[3]) ? magic[3] : '?');+ fprintf(stderr, "%s: invalid magic characters '%c%c%c%c'\n", __func__, magic[0], magic[1], magic[2], magic[3]);return NULL;}}@@ -19309,6 +19989,12 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_ireturn NULL;}+ // sanity-checks to prevent from integer/buffer overflows++ ok = ok && (ctx->header.n_tensors < (SIZE_MAX/2)/sizeof(struct gguf_tensor_info));+ ok = ok && (ctx->header.n_tensors < (SIZE_MAX/2)/ggml_tensor_overhead());+ ok = ok && (ctx->header.n_kv < (SIZE_MAX/2)/sizeof(struct gguf_kv));+if (!ok) {fprintf(stderr, "%s: failed to read header\n", __func__);gguf_free(ctx);@@ -19318,7 +20004,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i// read the kv pairs{- ctx->kv = malloc(ctx->header.n_kv * sizeof(struct gguf_kv));+ ctx->kv = GGML_MALLOC(ctx->header.n_kv * sizeof(struct gguf_kv));for (uint64_t i = 0; i < ctx->header.n_kv; ++i) {struct gguf_kv * kv = &ctx->kv[i];@@ -19346,7 +20032,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_icase GGUF_TYPE_ARRAY:{ok = ok && gguf_fread_el(file, &kv->value.arr.type, sizeof(kv->value.arr.type), &offset);- ok = ok && gguf_fread_el(file, &kv->value.arr.n, sizeof(kv->value.arr.n), &offset);+ ok = ok && gguf_fread_el(file, &kv->value.arr.n, sizeof(kv->value.arr.n), &offset);switch (kv->value.arr.type) {case GGUF_TYPE_UINT8:@@ -19361,21 +20047,37 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_icase GGUF_TYPE_FLOAT64:case GGUF_TYPE_BOOL:{- kv->value.arr.data = malloc(kv->value.arr.n * GGUF_TYPE_SIZE[kv->value.arr.type]);- ok = ok && gguf_fread_el(file, kv->value.arr.data, kv->value.arr.n * GGUF_TYPE_SIZE[kv->value.arr.type], &offset);+ // prevent from integer overflow in the malloc below+ if (kv->value.arr.n >= SIZE_MAX/gguf_type_size(kv->value.arr.type)) {+ fprintf(stderr, "%s: array size is too large (%" PRIu64 ")\n", __func__, kv->value.arr.n);+ gguf_free(ctx);+ return NULL;+ }++ kv->value.arr.data = GGML_MALLOC(kv->value.arr.n * gguf_type_size(kv->value.arr.type));++ ok = ok && gguf_fread_el(file, kv->value.arr.data, kv->value.arr.n * gguf_type_size(kv->value.arr.type), &offset);} break;case GGUF_TYPE_STRING:{- kv->value.arr.data = malloc(kv->value.arr.n * sizeof(struct gguf_str));+ // prevent from integer overflow in the malloc below+ if (kv->value.arr.n >= SIZE_MAX/sizeof(struct gguf_str)) {+ fprintf(stderr, "%s: array size is too large (%" PRIu64 ")\n", __func__, kv->value.arr.n);+ gguf_free(ctx);+ return NULL;+ }++ kv->value.arr.data = GGML_MALLOC(kv->value.arr.n * sizeof(struct gguf_str));+for (uint64_t j = 0; j < kv->value.arr.n; ++j) {ok = ok && gguf_fread_str(file, &((struct gguf_str *) kv->value.arr.data)[j], &offset);}} break;case GGUF_TYPE_ARRAY:- case GGUF_TYPE_COUNT: GGML_ASSERT(false && "invalid type"); break;+ default: GGML_ASSERT(false && "invalid type"); break;}} break;- case GGUF_TYPE_COUNT: GGML_ASSERT(false && "invalid type");+ default: GGML_ASSERT(false && "invalid type");}if (!ok) {@@ -19392,7 +20094,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i// read the tensor infos{- ctx->infos = malloc(ctx->header.n_tensors * sizeof(struct gguf_tensor_info));+ ctx->infos = GGML_MALLOC(ctx->header.n_tensors * sizeof(struct gguf_tensor_info));for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {struct gguf_tensor_info * info = &ctx->infos[i];@@ -19403,12 +20105,18 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_iok = ok && gguf_fread_str(file, &info->name, &offset);ok = ok && gguf_fread_el (file, &info->n_dims, sizeof(info->n_dims), &offset);++ ok = ok && (info->n_dims <= GGML_MAX_DIMS);+for (uint32_t j = 0; j < info->n_dims; ++j) {ok = ok && gguf_fread_el(file, &info->ne[j], sizeof(info->ne[j]), &offset);}+ok = ok && gguf_fread_el (file, &info->type, sizeof(info->type), &offset);ok = ok && gguf_fread_el (file, &info->offset, sizeof(info->offset), &offset);+ gguf_tensor_info_sanitize(info);+if (!ok) {fprintf(stderr, "%s: failed to read tensor info\n", __func__);gguf_free(ctx);@@ -19556,12 +20264,12 @@ void gguf_free(struct gguf_context * ctx) {struct gguf_kv * kv = &ctx->kv[i];if (kv->key.data) {- free(kv->key.data);+ GGML_FREE(kv->key.data);}if (kv->type == GGUF_TYPE_STRING) {if (kv->value.str.data) {- free(kv->value.str.data);+ GGML_FREE(kv->value.str.data);}}@@ -19571,16 +20279,16 @@ void gguf_free(struct gguf_context * ctx) {for (uint64_t j = 0; j < kv->value.arr.n; ++j) {struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[j];if (str->data) {- free(str->data);+ GGML_FREE(str->data);}}}- free(kv->value.arr.data);+ GGML_FREE(kv->value.arr.data);}}}- free(ctx->kv);+ GGML_FREE(ctx->kv);}if (ctx->infos) {@@ -19588,11 +20296,11 @@ void gguf_free(struct gguf_context * ctx) {struct gguf_tensor_info * info = &ctx->infos[i];if (info->name.data) {- free(info->name.data);+ GGML_FREE(info->name.data);}}- free(ctx->infos);+ GGML_FREE(ctx->infos);}GGML_ALIGNED_FREE(ctx);@@ -19893,8 +20601,8 @@ void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_tyctx->kv[idx].type = GGUF_TYPE_ARRAY;ctx->kv[idx].value.arr.type = type;ctx->kv[idx].value.arr.n = n;- ctx->kv[idx].value.arr.data = malloc(n*GGUF_TYPE_SIZE[type]);- memcpy(ctx->kv[idx].value.arr.data, data, n*GGUF_TYPE_SIZE[type]);+ ctx->kv[idx].value.arr.data = GGML_MALLOC(n*gguf_type_size(type));+ memcpy(ctx->kv[idx].value.arr.data, data, n*gguf_type_size(type));}void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char ** data, int n) {@@ -19903,7 +20611,7 @@ void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char **ctx->kv[idx].type = GGUF_TYPE_ARRAY;ctx->kv[idx].value.arr.type = GGUF_TYPE_STRING;ctx->kv[idx].value.arr.n = n;- ctx->kv[idx].value.arr.data = malloc(n*sizeof(struct gguf_str));+ ctx->kv[idx].value.arr.data = GGML_MALLOC(n*sizeof(struct gguf_str));for (int i = 0; i < n; i++) {struct gguf_str * str = &((struct gguf_str *)ctx->kv[idx].value.arr.data)[i];str->n = strlen(data[i]);@@ -19930,19 +20638,19 @@ void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {case GGUF_TYPE_ARRAY:{if (src->kv[i].value.arr.type == GGUF_TYPE_STRING) {- const char ** data = malloc(src->kv[i].value.arr.n*sizeof(char *));+ const char ** data = GGML_MALLOC(src->kv[i].value.arr.n*sizeof(char *));for (uint32_t j = 0; j < src->kv[i].value.arr.n; j++) {data[j] = ((struct gguf_str *)src->kv[i].value.arr.data)[j].data;}gguf_set_arr_str(ctx, src->kv[i].key.data, data, src->kv[i].value.arr.n);- free((void *)data);+ GGML_FREE((void *)data);} else if (src->kv[i].value.arr.type == GGUF_TYPE_ARRAY) {GGML_ASSERT(false && "nested arrays not supported");} else {gguf_set_arr_data(ctx, src->kv[i].key.data, src->kv[i].value.arr.type, src->kv[i].value.arr.data, src->kv[i].value.arr.n);}} break;- case GGUF_TYPE_COUNT: GGML_ASSERT(false && "invalid type"); break;+ default: GGML_ASSERT(false && "invalid type"); break;}}}@@ -20001,12 +20709,12 @@ void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const vo}}-//static void gguf_fwrite_str(struct llamafile * file, const struct gguf_str * val) {+//static void gguf_fwrite_str(FILE * file, const struct gguf_str * val) {// fwrite(&val->n, sizeof(val->n), 1, file);// fwrite(val->data, sizeof(char), val->n, file);//}//-//static void gguf_fwrite_el(struct llamafile * file, const void * val, size_t size) {+//static void gguf_fwrite_el(FILE * file, const void * val, size_t size) {// fwrite(val, sizeof(char), size, file);//}@@ -20018,7 +20726,7 @@ struct gguf_buf {static struct gguf_buf gguf_buf_init(size_t size) {struct gguf_buf buf = {- /*buf.data =*/ size == 0 ? NULL : malloc(size),+ /*buf.data =*/ size == 0 ? NULL : GGML_MALLOC(size),/*buf.size =*/ size,/*buf.offset =*/ 0,};@@ -20028,7 +20736,7 @@ static struct gguf_buf gguf_buf_init(size_t size) {static void gguf_buf_free(struct gguf_buf buf) {if (buf.data) {- free(buf.data);+ GGML_FREE(buf.data);}}@@ -20109,7 +20817,7 @@ static void gguf_write_to_buf(const struct gguf_context * ctx, struct gguf_buf *case GGUF_TYPE_FLOAT64:case GGUF_TYPE_BOOL:{- gguf_bwrite_el(buf, kv->value.arr.data, kv->value.arr.n * GGUF_TYPE_SIZE[kv->value.arr.type]);+ gguf_bwrite_el(buf, kv->value.arr.data, kv->value.arr.n * gguf_type_size(kv->value.arr.type));} break;case GGUF_TYPE_STRING:{@@ -20118,10 +20826,10 @@ static void gguf_write_to_buf(const struct gguf_context * ctx, struct gguf_buf *}} break;case GGUF_TYPE_ARRAY:- case GGUF_TYPE_COUNT: GGML_ASSERT(false && "invalid type"); break;+ default: GGML_ASSERT(false && "invalid type"); break;}} break;- case GGUF_TYPE_COUNT: GGML_ASSERT(false && "invalid type");+ default: GGML_ASSERT(false && "invalid type");}}@@ -20179,14 +20887,21 @@ static void gguf_write_to_buf(const struct gguf_context * ctx, struct gguf_buf *}}-void gguf_write_to_file(const struct gguf_context * ctx, struct llamafile * file, bool only_meta) {+void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta) {+ FILE * file = fopen(fname, "wb");+ if (!file) {+ GGML_ASSERT(false && "failed to open file for writing");+ }+struct gguf_buf buf = gguf_buf_init(16*1024);gguf_write_to_buf(ctx, &buf, only_meta);- llamafile_write(file, buf.data, buf.offset);+ fwrite(buf.data, 1, buf.offset, file);gguf_buf_free(buf);++ fclose(file);}size_t gguf_get_meta_size(const struct gguf_context * ctx) {@@ -20214,14 +20929,14 @@ int ggml_cpu_has_avx(void) {return X86_HAVE(AVX);}-int ggml_cpu_has_avx2(void) {- return X86_HAVE(AVX2);-}-int ggml_cpu_has_avx_vnni(void) {return X86_HAVE(AVXVNNI);}+int ggml_cpu_has_avx2(void) {+ return X86_HAVE(AVX2);+}+int ggml_cpu_has_avx512(void) {return X86_HAVE(AVX512F);}@@ -20294,8 +21009,33 @@ int ggml_cpu_has_clblast(void) {#endif}+int ggml_cpu_has_vulkan(void) {+#if defined(GGML_USE_VULKAN)+ return 1;+#else+ return 0;+#endif+}++int ggml_cpu_has_kompute(void) {+#if defined(GGML_USE_KOMPUTE)+ return 1;+#else+ return 0;+#endif+}++int ggml_cpu_has_sycl(void) {+#if defined(GGML_USE_SYCL)+ return 1;+#else+ return 0;+#endif+}+int ggml_cpu_has_gpublas(void) {- return ggml_cpu_has_cublas() || ggml_cpu_has_clblast();+ return ggml_cpu_has_cublas() || ggml_cpu_has_clblast() || ggml_cpu_has_vulkan() || ggml_cpu_has_kompute() ||+ ggml_cpu_has_sycl();}int ggml_cpu_has_sse3(void) {@@ -20314,4 +21054,12 @@ int ggml_cpu_has_vsx(void) {#endif}+int ggml_cpu_has_matmul_int8(void) {+#if defined(__ARM_FEATURE_MATMUL_INT8)+ return 1;+#else+ return 0;+#endif+}+////////////////////////////////////////////////////////////////////////////////diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex bda99c91..47002e46 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -356,6 +356,9 @@ extern "C" {GGML_TYPE_Q8_K = 15,GGML_TYPE_IQ2_XXS = 16,GGML_TYPE_IQ2_XS = 17,+ GGML_TYPE_IQ3_XXS = 18,+ GGML_TYPE_IQ1_S = 19,+ GGML_TYPE_IQ4_NL = 20,GGML_TYPE_I8,GGML_TYPE_I16,GGML_TYPE_I32,@@ -392,6 +395,9 @@ extern "C" {GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors};// available tensor operations:@@ -506,11 +512,17 @@ extern "C" {enum ggml_log_level {GGML_LOG_LEVEL_ERROR = 2,- GGML_LOG_LEVEL_WARN = 3,- GGML_LOG_LEVEL_INFO = 4,+ GGML_LOG_LEVEL_WARN = 3,+ GGML_LOG_LEVEL_INFO = 4,GGML_LOG_LEVEL_DEBUG = 5};+ enum ggml_tensor_flag {+ GGML_TENSOR_FLAG_INPUT = 1,+ GGML_TENSOR_FLAG_OUTPUT = 2,+ GGML_TENSOR_FLAG_PARAM = 4,+ };+// ggml objectstruct ggml_object {size_t offs;@@ -544,7 +556,7 @@ extern "C" {// op params - allocated as int32_t for alignmentint32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];- bool is_param;+ int32_t flags;struct ggml_tensor * grad;struct ggml_tensor * src[GGML_MAX_SRC];@@ -568,6 +580,11 @@ extern "C" {static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);+ // Abort callback+ // If not NULL, called before ggml computation+ // If it returns true, the computation is aborted+ typedef bool (*ggml_abort_callback)(void * data);+// the compute plan that needs to be prepared for ggml_graph_compute()// since https://github.com/ggerganov/ggml/issues/287struct ggml_cplan {@@ -577,8 +594,8 @@ extern "C" {int n_threads;// abort ggml_graph_compute when true- bool (*abort_callback)(void * data);- void * abort_callback_data;+ ggml_abort_callback abort_callback;+ void * abort_callback_data;};enum ggml_cgraph_eval_order {@@ -648,6 +665,16 @@ extern "C" {void * wdata;};+ // numa strategies+ enum ggml_numa_strategy {+ GGML_NUMA_STRATEGY_DISABLED = 0,+ GGML_NUMA_STRATEGY_DISTRIBUTE = 1,+ GGML_NUMA_STRATEGY_ISOLATE = 2,+ GGML_NUMA_STRATEGY_NUMACTL = 3,+ GGML_NUMA_STRATEGY_MIRROR = 4,+ GGML_NUMA_STRATEGY_COUNT+ };+// miscGGML_API void ggml_time_init(void); // call this once at the beginning of the program@@ -658,7 +685,7 @@ extern "C" {GGML_API void ggml_print_backtrace(void);- GGML_API void ggml_numa_init(void); // call once for better performance on NUMA systems+ GGML_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systemsGGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA nodeGGML_API void ggml_print_object (const struct ggml_object * obj);@@ -1363,13 +1390,17 @@ extern "C" {struct ggml_context * ctx,struct ggml_tensor * a);- // fused soft_max(a*scale + mask)+ // fused soft_max(a*scale + mask + pos[i]*(ALiBi slope))// mask is optional+ // pos is required when max_bias > 0.0f+ // max_bias = 0.0f for no ALiBiGGML_API struct ggml_tensor * ggml_soft_max_ext(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * mask,- float scale);+ struct ggml_tensor * pos,+ float scale,+ float max_bias);GGML_API struct ggml_tensor * ggml_soft_max_back(struct ggml_context * ctx,@@ -1471,12 +1502,13 @@ extern "C" {// alibi position embedding// in-place, returns view(a)- GGML_API struct ggml_tensor * ggml_alibi(+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_alibi(struct ggml_context * ctx,struct ggml_tensor * a,int n_past,int n_head,- float bias_max);+ float bias_max),+ "use ggml_soft_max_ext instead (will be removed in Mar 2024)");// clamp// in-place, returns view(a)@@ -1496,7 +1528,8 @@ extern "C" {int p1,int d0,int d1,- bool is_2D);+ bool is_2D,+ enum ggml_type dst_type);GGML_API struct ggml_tensor * ggml_conv_depthwise_2d(struct ggml_context * ctx,@@ -2087,6 +2120,12 @@ extern "C" {ggml_opt_callback callback,void * callback_data);+ //+ // tensor flags+ //+ GGML_API void ggml_set_input(struct ggml_tensor * tensor);+ GGML_API void ggml_set_output(struct ggml_tensor * tensor);+//// quantization//@@ -2240,7 +2279,7 @@ extern "C" {//// write the entire context to a binary file- GGML_API void gguf_write_to_file(const struct gguf_context * ctx, struct llamafile * file, bool only_meta);+ GGML_API void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);// get the size in bytes of the meta data (header, kv pairs, tensor info) including paddingGGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);@@ -2266,10 +2305,14 @@ extern "C" {GGML_API int ggml_cpu_has_blas (void);GGML_API int ggml_cpu_has_cublas (void);GGML_API int ggml_cpu_has_clblast (void);+ GGML_API int ggml_cpu_has_vulkan (void);+ GGML_API int ggml_cpu_has_kompute (void);GGML_API int ggml_cpu_has_gpublas (void);GGML_API int ggml_cpu_has_sse3 (void);GGML_API int ggml_cpu_has_ssse3 (void);+ GGML_API int ggml_cpu_has_sycl (void);GGML_API int ggml_cpu_has_vsx (void);+ GGML_API int ggml_cpu_has_matmul_int8(void);//// Internal types and functions exposed for tests and benchmarks@@ -2283,7 +2326,8 @@ extern "C" {#endiftypedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);- typedef void (*ggml_vec_dot_t) (const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT x, const void * GGML_RESTRICT y);+ typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,+ const void * GGML_RESTRICT y, size_t by, int nrc);typedef struct {const char * type_name;@@ -2295,6 +2339,7 @@ extern "C" {ggml_from_float_t from_float_reference;ggml_vec_dot_t vec_dot;enum ggml_type vec_dot_type;+ int64_t nrows; // number of rows to process simultaneously;} ggml_type_traits_t;GGML_API ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);diff --git a/llama.cpp/ggml_vec_dot_iq2_xs_q8_K.inc b/llama.cpp/ggml_vec_dot_iq2_xs_q8_K.incdeleted file mode 100644index fd7eca9f..00000000--- a/llama.cpp/ggml_vec_dot_iq2_xs_q8_K.inc+++ /dev/null@@ -1,157 +0,0 @@-static void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- assert(n % QK_K == 0);-- const block_iq2_xs * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;--#if defined(__ARM_NEON)-- const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;-- ggml_int8x16x4_t q2u;- ggml_int8x16x4_t q2s;- ggml_int8x16x4_t q8b;-- int32x4x4_t scales32;-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- const uint16_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- const uint8x8_t scales8 = vld1_u8(x[i].scales);- const uint8x8_t scales_l = vand_u8(scales8, vdup_n_u8(0xf));- const uint8x8_t scales_h = vshr_n_u8(scales8, 4);- uint8x16_t scales = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h));- scales = vaddq_u8(vshlq_n_u8(scales, 1), vdupq_n_u8(1));- const uint16x8_t scales1 = vmovl_u8(vget_low_u8(scales));- const uint16x8_t scales2 = vmovl_u8(vget_high_u8(scales));- scales32.val[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales1)));- scales32.val[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales1)));- scales32.val[2] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales2)));- scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2)));- int32x4_t sumi = vdupq_n_s32(0);- for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {- q8b = ggml_vld1q_s8_x4(q8); q8 += 64;- q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511))));- q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511))));- q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511))));- q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[6] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[7] & 511))));- q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[0] >> 9))), vld1_s8((const void *)(signs64 + (q2[1] >> 9))));- q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[2] >> 9))), vld1_s8((const void *)(signs64 + (q2[3] >> 9))));- q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[4] >> 9))), vld1_s8((const void *)(signs64 + (q2[5] >> 9))));- q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[6] >> 9))), vld1_s8((const void *)(signs64 + (q2[7] >> 9))));- q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);- q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);- q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);- q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);- const int32x4_t p1 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]);- const int32x4_t p2 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[1], q8b.val[1]);- const int32x4_t p3 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]);- const int32x4_t p4 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[3], q8b.val[3]);- const int32x4_t p = vpaddq_s32(vpaddq_s32(p1, p2), vpaddq_s32(p3, p4));- sumi = vmlaq_s32(sumi, p, scales32.val[ib64]);- q2 += 8;- }- sumf += d*vaddvq_s32(sumi);- }- *s = 0.125f * sumf;--#elif defined(__AVX2__)-- const __m128i m4 = _mm_set1_epi8(0xf);- const __m128i m1 = _mm_set1_epi8(1);- const __m128i m511 = _mm_set1_epi16(511);- const __m128i m127 = _mm_set1_epi16(127);-- const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;-- uint64_t aux64;-- // somewhat hacky, but gives a significant boost in performance- __m128i aux_gindex, aux_sindex;- const uint16_t * gindex = (const uint16_t *)&aux_gindex;- const uint16_t * sindex = (const uint16_t *)&aux_sindex;-- __m256 accumf = _mm256_setzero_ps();- for (int i = 0; i < nb; ++i) {- const float d = GGML_FP16_TO_FP32__F16C(x[i].d) * y[i].d;- const uint16_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- memcpy(&aux64, x[i].scales, 8);- __m128i stmp = _mm_set1_epi64x(aux64);- stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));- const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);-- __m256i sumi1 = _mm256_setzero_si256();- __m256i sumi2 = _mm256_setzero_si256();- for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {- const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- const __m128i q2_data = _mm_loadu_si128((const __m128i*)q2); q2 += 8;- aux_gindex = _mm_and_si128(q2_data, m511);- aux_sindex = _mm_and_si128(_mm_srli_epi16(q2_data, 9), m127);- const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[3]], iq2xs_grid[gindex[2]], iq2xs_grid[gindex[1]], iq2xs_grid[gindex[0]]);- const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[7]], iq2xs_grid[gindex[6]], iq2xs_grid[gindex[5]], iq2xs_grid[gindex[4]]);- const __m256i s2_1 = _mm256_set_epi64x(signs64[sindex[3]], signs64[sindex[2]], signs64[sindex[1]], signs64[sindex[0]]);- const __m256i s2_2 = _mm256_set_epi64x(signs64[sindex[7]], signs64[sindex[6]], signs64[sindex[5]], signs64[sindex[4]]);- const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);- const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);- const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);- const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);-- const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0)));- const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1)));-- sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1));- sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2));- }-- accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);-- }-- *s = 0.125f * hsum_float_8(accumf);--#else-- float sumf = 0.f;- for (int i = 0; i < nb; ++i) {- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- const uint16_t * restrict q2 = x[i].qs;- const uint8_t * restrict sc = x[i].scales;- const int8_t * restrict q8 = y[i].qs;- int32_t bsum = 0;- for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {- const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;- const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1;- int32_t sumi = 0;- for (int l = 0; l < 2; ++l) {- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];- for (int j = 0; j < 8; ++j) {- sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- }- bsum += sumi * ls1;- sumi = 0;- for (int l = 2; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];- for (int j = 0; j < 8; ++j) {- sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- }- bsum += sumi * ls2;- q2 += 4;- }- sumf += d * bsum;- }- *s = 0.125f * sumf;-#endif-}diff --git a/llama.cpp/ggml_vec_dot_iq2_xxs_q8_K.inc b/llama.cpp/ggml_vec_dot_iq2_xxs_q8_K.incdeleted file mode 100644index 4a237314..00000000--- a/llama.cpp/ggml_vec_dot_iq2_xxs_q8_K.inc+++ /dev/null@@ -1,122 +0,0 @@-static void ggml_vec_dot_iq2_xxs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- assert(n % QK_K == 0);-- const block_iq2_xxs * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;--#if defined(__ARM_NEON)-- const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;-- uint32_t aux32[4];- const uint8_t * aux8 = (const uint8_t *)aux32;-- ggml_int8x16x4_t q2u;- ggml_int8x16x4_t q2s;- ggml_int8x16x4_t q8b;-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- const uint16_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- float sumf1 = 0, sumf2 = 0;- for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {- q8b = ggml_vld1q_s8_x4(q8); q8 += 64;- memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;- q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 0])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 1])));- q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 2])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 3])));- q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 8])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 9])));- q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[10])), vld1_s8((const void *)(iq2xxs_grid + aux8[11])));- q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 7) & 127))));- q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));- q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 7) & 127))));- q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 21) & 127))));- q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);- q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);- q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);- q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);- const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]), q2u.val[1], q8b.val[1]);- const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]), q2u.val[3], q8b.val[3]);- sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28));- sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[3] >> 28));- }- sumf += d*(sumf1 + sumf2);- }- *s = 0.25f * sumf;--#elif defined(__AVX2__)-- const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;-- uint32_t aux32[4];- const uint8_t * aux8 = (const uint8_t *)aux32;-- __m256 accumf = _mm256_setzero_ps();- for (int i = 0; i < nb; ++i) {- const float d = GGML_FP16_TO_FP32__F16C(x[i].d) * y[i].d;- const uint16_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- __m256i sumi1 = _mm256_setzero_si256();- __m256i sumi2 = _mm256_setzero_si256();- for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {- const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;- const __m256i q2_1 = _mm256_set_epi64x(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);- const __m256i q2_2 = _mm256_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);- const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],- signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);- const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],- signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);- const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);- const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);- const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);- const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);- const uint16_t ls1 = aux32[1] >> 28;- const uint16_t ls2 = aux32[3] >> 28;- const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));- const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));- sumi1 = _mm256_add_epi32(sumi1, p1);- sumi2 = _mm256_add_epi32(sumi2, p2);- }-- accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);-- }-- *s = 0.125f * hsum_float_8(accumf);--#else-- uint32_t aux32[2];- const uint8_t * aux8 = (const uint8_t *)aux32;-- float sumf = 0.f;- for (int i = 0; i < nb; ++i) {- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- const uint16_t * restrict q2 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- int32_t bsum = 0;- for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {- memcpy(aux32, q2, 2*sizeof(uint32_t));- q2 += 4;- const uint32_t ls = 2*(aux32[1] >> 28) + 1;- int32_t sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);- const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];- for (int j = 0; j < 8; ++j) {- sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- }- bsum += sumi * ls;- }- sumf += d * bsum;- }- *s = 0.125f * sumf;-#endif-}diff --git a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.incindex c5ae1427..ebbaa970 100644--- a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const block_q2_K * restrict x = vx;const block_q8_K * restrict y = vy;diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.incindex 3019db38..aaf501f2 100644--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const uint32_t kmask1 = 0x03030303;diff --git a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q4_0_q8_0.incindex dd7e2272..822395d5 100644--- a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const int qk = QK8_0;const int nb = n / qk;diff --git a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q4_1_q8_1.incindex d48aff36..07ab3038 100644--- a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc+++ b/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const int qk = QK8_1;const int nb = n / qk;diff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.incindex e0933031..589fe471 100644--- a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const block_q4_K * restrict x = vx;diff --git a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q5_0_q8_0.incindex 54fae8ce..8ba7bae9 100644--- a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const int qk = QK8_0;const int nb = n / qk;diff --git a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q5_1_q8_1.incindex 8d79b71a..bc77430a 100644--- a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc+++ b/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const int qk = QK8_1;const int nb = n / qk;diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.incindex b75aad44..eb8852fa 100644--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const block_q5_K * restrict x = vx;diff --git a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.incindex e5d8f6c9..091d592a 100644--- a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);const block_q6_K * restrict x = vx;diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.incindex 1800acfe..c90ee156 100644--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc@@ -1,4 +1,4 @@-static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {const int qk = QK8_0;const int nb = n / qk;diff --git a/llama.cpp/imatrix/imatrix.cpp b/llama.cpp/imatrix/imatrix.cppindex 8eebefc7..088db919 100644--- a/llama.cpp/imatrix/imatrix.cpp+++ b/llama.cpp/imatrix/imatrix.cpp@@ -40,6 +40,8 @@ public:void set_parameters(StatParams&& params) { m_params = std::move(params); }bool collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data);void save_imatrix() const;+ bool load_imatrix(const char * file_name, bool add);+ static bool load_imatrix(const char * file_name, std::unordered_map<std::string, Stats>& imatrix);private:std::unordered_map<std::string, Stats> m_stats;StatParams m_params;@@ -193,6 +195,57 @@ void IMatrixCollector::save_imatrix(const char * fname) const {}}+bool IMatrixCollector::load_imatrix(const char * imatrix_file, std::unordered_map<std::string, Stats>& imatrix_data) {+ std::ifstream in(imatrix_file, std::ios::binary);+ if (!in) {+ printf("%s: failed to open %s\n",__func__,imatrix_file);+ return false;+ }+ int n_entries;+ in.read((char*)&n_entries, sizeof(n_entries));+ if (in.fail() || n_entries < 1) {+ printf("%s: no data in file %s\n", __func__, imatrix_file);+ return false;+ }+ for (int i = 0; i < n_entries; ++i) {+ int len; in.read((char *)&len, sizeof(len));+ std::vector<char> name_as_vec(len+1);+ in.read((char *)name_as_vec.data(), len);+ if (in.fail()) {+ printf("%s: failed reading name for entry %d from %s\n",__func__,i+1,imatrix_file);+ return false;+ }+ name_as_vec[len] = 0;+ std::string name{name_as_vec.data()};+ auto& e = imatrix_data[std::move(name)];+ int ncall;+ in.read((char*)&ncall, sizeof(ncall));+ int nval;+ in.read((char *)&nval, sizeof(nval));+ if (in.fail() || nval < 1) {+ printf("%s: failed reading number of values for entry %d\n",__func__,i);+ imatrix_data = {};+ return false;+ }+ e.values.resize(nval);+ in.read((char*)e.values.data(), nval*sizeof(float));+ if (in.fail()) {+ printf("%s: failed reading data for entry %d\n",__func__,i);+ imatrix_data = {};+ return false;+ }+ e.ncall = ncall;+ }+ return true;+}++bool IMatrixCollector::load_imatrix(const char * file_name, bool add) {+ if (!add) {+ m_stats.clear();+ }+ return load_imatrix(file_name, m_stats);+}+static IMatrixCollector g_collector;static bool ik_collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) {@@ -273,7 +326,7 @@ static void process_logits(}}-static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool compute_ppl) {+static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool compute_ppl, int from_chunk) {const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));const int n_ctx = llama_n_ctx(ctx);@@ -286,6 +339,15 @@ static bool compute_imatrix(llama_context * ctx, const gpt_params & params, boolauto tim2 = std::chrono::high_resolution_clock::now();fprintf(stderr, "%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast<std::chrono::microseconds>(tim2-tim1).count());+ if (from_chunk > 0) {+ if (size_t((from_chunk + 2)*n_ctx) >= tokens.size()) {+ fprintf(stderr, "%s: there will be not enough tokens left after removing %d chunks\n", __func__, from_chunk);+ return false;+ }+ fprintf(stderr, "%s: removing initial %d chunks (%d tokens)\n", __func__, from_chunk, from_chunk*n_ctx);+ tokens.erase(tokens.begin(), tokens.begin() + from_chunk*n_ctx);+ }+if (int(tokens.size()) < 2*n_ctx) {fprintf(stderr, "%s: you need at least %d tokens for a context of %d tokens\n",__func__,2*n_ctx,n_ctx);@@ -421,7 +483,10 @@ int main(int argc, char ** argv) {llamafile_check_cpu();StatParams sparams;+ std::string prev_result_file;+ std::string combine_files;bool compute_ppl = true;+ int from_chunk = 0;std::vector<char*> args;args.push_back(argv[0]);int iarg = 1;@@ -442,6 +507,13 @@ int main(int argc, char ** argv) {compute_ppl = false;} else if (arg == "--keep-imatrix") {sparams.keep_every = std::stoi(argv[++iarg]);+ } else if (arg == "--continue-from") {+ prev_result_file = argv[++iarg];+ } else if (arg == "--combine") {+ combine_files = argv[++iarg];+ }+ else if (arg == "--from-chunk") {+ from_chunk = std::stoi(argv[++iarg]);} else {args.push_back(argv[iarg]);}@@ -455,14 +527,50 @@ int main(int argc, char ** argv) {}}+ g_collector.set_parameters(std::move(sparams));++ if (!combine_files.empty()) {+ std::vector<std::string> files;+ size_t pos = 0;+ while (true) {+ auto new_pos = combine_files.find(',', pos);+ if (new_pos != std::string::npos) {+ files.emplace_back(combine_files.substr(pos, new_pos - pos));+ pos = new_pos + 1;+ } else {+ files.emplace_back(combine_files.substr(pos));+ break;+ }+ }+ if (files.size() < 2) {+ fprintf(stderr, "You must provide at least two comma separated files to use --combine\n");+ return 1;+ }+ printf("Combining the following %d files\n", int(files.size()));+ for (auto& file : files) {+ printf(" %s\n", file.c_str());+ if (!g_collector.load_imatrix(file.c_str(), true)) {+ fprintf(stderr, "Failed to load %s\n", file.c_str());+ return 1;+ }+ }+ g_collector.save_imatrix();+ return 0;+ }++ if (!prev_result_file.empty()) {+ if (!g_collector.load_imatrix(prev_result_file.c_str(), false)) {+ fprintf(stderr, "=============== Failed to load %s\n", prev_result_file.c_str());+ return 1;+ }+ }+gpt_params params;params.n_batch = 512;if (!gpt_params_parse(args.size(), args.data(), params)) {return 1;}- g_collector.set_parameters(std::move(sparams));-params.logits_all = true;params.n_batch = std::min(params.n_batch, params.n_ctx);@@ -479,7 +587,8 @@ int main(int argc, char ** argv) {params.prompt = gpt_random_prompt(rng);}- llama_backend_init(params.numa);+ llama_backend_init();+ llama_numa_init(params.numa);llama_model_params mparams = llama_model_params_from_gpt_params(params);@@ -514,7 +623,7 @@ int main(int argc, char ** argv) {fprintf(stderr, "%s\n", get_system_info(params).c_str());}- bool OK = compute_imatrix(ctx, params, compute_ppl);+ bool OK = compute_imatrix(ctx, params, compute_ppl, from_chunk);if (!OK) {return 1;}diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 5b01de07..1667cd56 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -54,7 +54,7 @@#include <sys/mman.h>#include <sys/resource.h>-#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))+#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((__format__(__gnu_printf__, __VA_ARGS__)))#define LLAMA_MAX_NODES 8192#define LLAMA_MAX_EXPERTS 8@@ -153,6 +153,8 @@ enum llm_arch {LLM_ARCH_STARCODER,LLM_ARCH_PERSIMMON,LLM_ARCH_REFACT,+ LLM_ARCH_BERT,+ LLM_ARCH_NOMIC_BERT,LLM_ARCH_BLOOM,LLM_ARCH_STABLELM,LLM_ARCH_QWEN,@@ -160,27 +162,37 @@ enum llm_arch {LLM_ARCH_PHI2,LLM_ARCH_PLAMO,LLM_ARCH_CODESHELL,+ LLM_ARCH_ORION,+ LLM_ARCH_INTERNLM2,+ LLM_ARCH_MINICPM,+ LLM_ARCH_GEMMA,LLM_ARCH_UNKNOWN,};-static std::map<llm_arch, std::string> LLM_ARCH_NAMES = {- { LLM_ARCH_LLAMA, "llama" },- { LLM_ARCH_FALCON, "falcon" },- { LLM_ARCH_GPT2, "gpt2" },- { LLM_ARCH_GPTJ, "gptj" },- { LLM_ARCH_GPTNEOX, "gptneox" },- { LLM_ARCH_MPT, "mpt" },- { LLM_ARCH_BAICHUAN, "baichuan" },- { LLM_ARCH_STARCODER, "starcoder" },- { LLM_ARCH_PERSIMMON, "persimmon" },- { LLM_ARCH_REFACT, "refact" },- { LLM_ARCH_BLOOM, "bloom" },- { LLM_ARCH_STABLELM, "stablelm" },- { LLM_ARCH_QWEN, "qwen" },- { LLM_ARCH_QWEN2, "qwen2" },- { LLM_ARCH_PHI2, "phi2" },- { LLM_ARCH_PLAMO, "plamo" },- { LLM_ARCH_CODESHELL, "codeshell" },+static std::map<llm_arch, const char *> LLM_ARCH_NAMES = {+ { LLM_ARCH_LLAMA, "llama" },+ { LLM_ARCH_FALCON, "falcon" },+ { LLM_ARCH_GPT2, "gpt2" },+ { LLM_ARCH_GPTJ, "gptj" },+ { LLM_ARCH_GPTNEOX, "gptneox" },+ { LLM_ARCH_MPT, "mpt" },+ { LLM_ARCH_BAICHUAN, "baichuan" },+ { LLM_ARCH_STARCODER, "starcoder" },+ { LLM_ARCH_PERSIMMON, "persimmon" },+ { LLM_ARCH_REFACT, "refact" },+ { LLM_ARCH_BERT, "bert" },+ { LLM_ARCH_NOMIC_BERT, "nomic-bert" },+ { LLM_ARCH_BLOOM, "bloom" },+ { LLM_ARCH_STABLELM, "stablelm" },+ { LLM_ARCH_QWEN, "qwen" },+ { LLM_ARCH_QWEN2, "qwen2" },+ { LLM_ARCH_PHI2, "phi2" },+ { LLM_ARCH_PLAMO, "plamo" },+ { LLM_ARCH_CODESHELL, "codeshell" },+ { LLM_ARCH_ORION, "orion" },+ { LLM_ARCH_INTERNLM2, "internlm2" },+ { LLM_ARCH_MINICPM, "minicpm" },+ { LLM_ARCH_GEMMA, "gemma" },};enum llm_kv {@@ -203,6 +215,7 @@ enum llm_kv {LLM_KV_TENSOR_DATA_LAYOUT,LLM_KV_EXPERT_COUNT,LLM_KV_EXPERT_USED_COUNT,+ LLM_KV_POOLING_TYPE,LLM_KV_ATTENTION_HEAD_COUNT,LLM_KV_ATTENTION_HEAD_COUNT_KV,@@ -212,6 +225,7 @@ enum llm_kv {LLM_KV_ATTENTION_VALUE_LENGTH,LLM_KV_ATTENTION_LAYERNORM_EPS,LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,+ LLM_KV_ATTENTION_CAUSAL,LLM_KV_ROPE_DIMENSION_COUNT,LLM_KV_ROPE_FREQ_BASE,@@ -224,6 +238,7 @@ enum llm_kv {LLM_KV_TOKENIZER_MODEL,LLM_KV_TOKENIZER_LIST,LLM_KV_TOKENIZER_TOKEN_TYPE,+ LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,LLM_KV_TOKENIZER_SCORES,LLM_KV_TOKENIZER_MERGES,LLM_KV_TOKENIZER_BOS_ID,@@ -233,11 +248,12 @@ enum llm_kv {LLM_KV_TOKENIZER_PAD_ID,LLM_KV_TOKENIZER_ADD_BOS,LLM_KV_TOKENIZER_ADD_EOS,+ LLM_KV_TOKENIZER_ADD_PREFIX,LLM_KV_TOKENIZER_HF_JSON,LLM_KV_TOKENIZER_RWKV,};-static std::map<llm_kv, std::string> LLM_KV_NAMES = {+static std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" },{ LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" },{ LLM_KV_GENERAL_ALIGNMENT, "general.alignment" },@@ -257,6 +273,7 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {{ LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" },{ LLM_KV_EXPERT_COUNT, "%s.expert_count" },{ LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" },+ { LLM_KV_POOLING_TYPE , "%s.pooling_type" },{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },@@ -266,6 +283,7 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {{ LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },{ LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },{ LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },+ { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" },{ LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },{ LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },@@ -278,6 +296,7 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },+ { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },{ LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },{ LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },{ LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },@@ -287,6 +306,7 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {{ LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },{ LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },{ LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },+ { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },};@@ -297,13 +317,14 @@ struct LLM_KV {llm_arch arch;std::string operator()(llm_kv kv) const {- return ::format(LLM_KV_NAMES[kv].c_str(), LLM_ARCH_NAMES[arch].c_str());+ return ::format(LLM_KV_NAMES[kv], LLM_ARCH_NAMES[arch]);}};enum llm_tensor {LLM_TENSOR_TOKEN_EMBD,LLM_TENSOR_TOKEN_EMBD_NORM,+ LLM_TENSOR_TOKEN_TYPES,LLM_TENSOR_POS_EMBD,LLM_TENSOR_OUTPUT,LLM_TENSOR_OUTPUT_NORM,@@ -315,6 +336,7 @@ enum llm_tensor {LLM_TENSOR_ATTN_OUT,LLM_TENSOR_ATTN_NORM,LLM_TENSOR_ATTN_NORM_2,+ LLM_TENSOR_ATTN_OUT_NORM,LLM_TENSOR_ATTN_ROT_EMBD,LLM_TENSOR_FFN_GATE_INP,LLM_TENSOR_FFN_NORM,@@ -327,6 +349,7 @@ enum llm_tensor {LLM_TENSOR_FFN_UP_EXP,LLM_TENSOR_ATTN_Q_NORM,LLM_TENSOR_ATTN_K_NORM,+ LLM_TENSOR_LAYER_OUT_NORM,};static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {@@ -485,6 +508,38 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_BERT,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },+ { LLM_TENSOR_TOKEN_TYPES, "token_types" },+ { LLM_TENSOR_POS_EMBD, "position_embd" },+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },+ {+ LLM_ARCH_NOMIC_BERT,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },+ { LLM_TENSOR_TOKEN_TYPES, "token_types" },+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_BLOOM,{@@ -605,7 +660,81 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},-+ {+ LLM_ARCH_ORION,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },+ {+ LLM_ARCH_INTERNLM2,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },+ {+ LLM_ARCH_MINICPM,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" },+ { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" },+ { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" },+ },+ },+ {+ LLM_ARCH_GEMMA,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_UNKNOWN,{@@ -639,22 +768,37 @@ struct LLM_TN {llm_arch arch;std::string operator()(llm_tensor tensor) const {+ if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ return "__missing__";+ }return LLM_TENSOR_NAMES[arch].at(tensor);}std::string operator()(llm_tensor tensor, const std::string & suffix) const {+ if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ return "__missing__";+ }return LLM_TENSOR_NAMES[arch].at(tensor) + "." + suffix;}std::string operator()(llm_tensor tensor, int bid) const {+ if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ return "__missing__";+ }return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid);}std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const {+ if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ return "__missing__";+ }return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix;}std::string operator()(llm_tensor tensor, const std::string & suffix, int bid, int xid) const {+ if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ return "__missing__";+ }return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid, xid) + "." + suffix;}};@@ -663,13 +807,13 @@ struct LLM_TN {// gguf helpers//-static std::map<int8_t, std::string> LLAMA_ROPE_SCALING_TYPES = {+static std::map<int32_t, const char *> LLAMA_ROPE_SCALING_TYPES = {{ LLAMA_ROPE_SCALING_NONE, "none" },{ LLAMA_ROPE_SCALING_LINEAR, "linear" },{ LLAMA_ROPE_SCALING_YARN, "yarn" },};-static int8_t llama_rope_scaling_type_from_string(const std::string & name) {+static int32_t llama_rope_scaling_type_from_string(const std::string & name) {for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {if (kv.second == name) {return kv.first;@@ -1052,10 +1196,20 @@ static std::string llama_token_to_piece(const struct llama_context * ctx, llama_static ggml_backend_buffer_type_t llama_default_buffer_type_cpu(bool host_buffer) {ggml_backend_buffer_type_t buft = nullptr;+// #if defined(GGML_USE_CUBLAS)// host buffers should only be used when data is expected to be copied to/from the GPUif (host_buffer) {buft = ggml_backend_cuda_host_buffer_type();}+// #elif defined(GGML_USE_SYCL)+// buft = ggml_backend_sycl_host_buffer_type();+// #elif defined(GGML_USE_CPU_HBM)+// buft = ggml_backend_cpu_hbm_buffer_type();+// #elif defined(GGML_USE_VULKAN)+// if (host_buffer) {+// buft = ggml_backend_vk_host_buffer_type();+// }+// #endifif (buft == nullptr) {buft = ggml_backend_cpu_buffer_type();@@ -1070,6 +1224,22 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_offload(int gpu) {if (!buft) buft = ggml_backend_metal_buffer_type();if (!buft) buft = ggml_backend_cuda_buffer_type(gpu);+// #ifdef GGML_USE_METAL+// buft = ggml_backend_metal_buffer_type();+// #elif defined(GGML_USE_CUBLAS)+// buft = ggml_backend_cuda_buffer_type(gpu);+// #elif defined(GGML_USE_VULKAN)+// buft = ggml_backend_vk_buffer_type(gpu);+// #elif defined(GGML_USE_SYCL)+// buft = ggml_backend_sycl_buffer_type(gpu);+// #elif defined(GGML_USE_CLBLAST)+// buft = ggml_backend_opencl_buffer_type();+// #elif defined(GGML_USE_KOMPUTE)+// buft = ggml_backend_kompute_buffer_type(gpu);+// if (buft == nullptr) {+// LLAMA_LOG_WARN("%s: cannot use GPU %d, check `vulkaninfo --summary`\n", __func__, gpu);+// }+// #endifif (buft == nullptr) {buft = llama_default_buffer_type_cpu(true);@@ -1094,6 +1264,41 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_gGGML_UNUSED(tensor_split);}+static size_t llama_get_device_count() {+ size_t res = ggml_backend_cuda_get_device_count();+ if (!res) res = 1;+ return res;+// #if defined(GGML_USE_CUBLAS)+// return ggml_backend_cuda_get_device_count();+// #elif defined(GGML_USE_VULKAN)+// return ggml_backend_vk_get_device_count();+// #else+// return 1;+// #endif+}++static size_t llama_get_device_memory(int device) {+ size_t total = 0;+ size_t free = 0;+ ggml_backend_cuda_get_device_memory(device, &total, &free);+ if (!free) free = 1;+ return free;+// #if defined(GGML_USE_CUBLAS)+// size_t total;+// size_t free;+// ggml_backend_cuda_get_device_memory(device, &total, &free);+// return free;+// #elif defined(GGML_USE_VULKAN)+// size_t total;+// size_t free;+// ggml_backend_vk_get_device_memory(device, &total, &free);+// return free;+// #else+// return 1;+// GGML_UNUSED(device);+// #endif+}+//// globals//@@ -1101,7 +1306,9 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_gstruct llama_state {llama_state() {// [jart] don't load dso early in initialization- // ggml_metal_log_set_callback(log_callback, log_callback_user_data);+// #ifdef GGML_USE_METAL+// ggml_backend_metal_log_set_callback(log_callback, log_callback_user_data);+// #endif}// We save the log callback globally@@ -1114,14 +1321,23 @@ static llama_state g_state;// available llama modelsenum e_model {MODEL_UNKNOWN,+ MODEL_17M,+ MODEL_22M,+ MODEL_33M,+ MODEL_109M,+ MODEL_137M,+ MODEL_335M,MODEL_0_5B,MODEL_1B,+ MODEL_2B,MODEL_3B,MODEL_4B,MODEL_7B,MODEL_8B,MODEL_13B,+ MODEL_14B,MODEL_15B,+ MODEL_20B,MODEL_30B,MODEL_34B,MODEL_40B,@@ -1139,6 +1355,7 @@ static const size_t GiB = 1024*MiB;struct llama_hparams {bool vocab_only;+ bool rope_finetuned;uint32_t n_vocab;uint32_t n_ctx_train; // context size the model was trained onuint32_t n_embd;@@ -1151,6 +1368,7 @@ struct llama_hparams {uint32_t n_ff;uint32_t n_expert = 0;uint32_t n_expert_used = 0;+ uint32_t n_vocab_type = 0; // for BERT-style token typesfloat f_norm_eps;float f_norm_rms_eps;@@ -1158,12 +1376,15 @@ struct llama_hparams {float rope_freq_base_train;float rope_freq_scale_train;uint32_t n_yarn_orig_ctx;- int8_t rope_scaling_type_train : 3;- bool rope_finetuned : 1;+ int32_t rope_scaling_type_train;++ float f_clamp_kqv = 0.0f;+ float f_max_alibi_bias = 0.0f;- float f_clamp_kqv;- float f_max_alibi_bias;+ bool causal_attn = true;+ bool need_kq_pos = false;+ uint32_t pooling_type = LLAMA_POOLING_NONE;bool operator!=(const llama_hparams & other) const {if (this->vocab_only != other.vocab_only) return true;@@ -1225,6 +1446,7 @@ struct llama_cparams {bool mul_mat_q;bool offload_kqv;+ bool do_pooling;ggml_backend_sched_eval_callback cb_eval;void * cb_eval_user_data;@@ -1240,6 +1462,8 @@ struct llama_layer {struct ggml_tensor * attn_q_norm_b;struct ggml_tensor * attn_k_norm;struct ggml_tensor * attn_k_norm_b;+ struct ggml_tensor * attn_out_norm;+ struct ggml_tensor * attn_out_norm_b;// attentionstruct ggml_tensor * wq;@@ -1258,6 +1482,8 @@ struct llama_layer {// normalizationstruct ggml_tensor * ffn_norm;struct ggml_tensor * ffn_norm_b;+ struct ggml_tensor * layer_out_norm;+ struct ggml_tensor * layer_out_norm_b;// ffstruct ggml_tensor * ffn_gate; // w1@@ -1363,6 +1589,8 @@ struct llama_vocab {id special_suffix_id = 32008;id special_eot_id = 32010;+ bool add_space_prefix = true;+int find_bpe_rank(const std::string & token_left, const std::string & token_right) const {GGML_ASSERT(token_left.find(' ') == std::string::npos);GGML_ASSERT(token_left.find('\n') == std::string::npos);@@ -1389,6 +1617,7 @@ struct llama_model {llama_vocab vocab;struct ggml_tensor * tok_embd;+ struct ggml_tensor * type_embd;struct ggml_tensor * pos_embd;struct ggml_tensor * tok_norm;struct ggml_tensor * tok_norm_b;@@ -1459,6 +1688,10 @@ struct llama_context {ggml_backend_free(backend);}+#ifdef GGML_USE_VULKAN+ ggml_vk_free_cpu_assist();+#endif+ggml_backend_buffer_free(buf_input);ggml_free(ctx_input);}@@ -1502,8 +1735,6 @@ struct llama_context {// memory buffers used to evaluate the modelstd::vector<uint8_t> buf_compute_meta;ggml_backend_sched_t sched = nullptr;- // allocator for the input tensors- ggml_tallocr * alloc = nullptr;// input tensorsggml_backend_buffer_t buf_input = nullptr;@@ -1512,7 +1743,10 @@ struct llama_context {struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch]struct ggml_tensor * inp_pos; // I32 [n_batch]struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch]+ struct ggml_tensor * inp_KQ_pos; // F32 [n_ctx]struct ggml_tensor * inp_K_shift; // I32 [n_ctx]+ struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch]+ struct ggml_tensor * inp_cls; // I32 [n_batch]#ifdef GGML_USE_MPIggml_mpi_context * ctx_mpi = NULL;@@ -2111,6 +2345,9 @@ struct llama_model_loader {case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;+ case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;+ case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;+ case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;default:{LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));@@ -2356,13 +2593,7 @@ struct llama_model_loader {std::vector<no_init<uint8_t>> read_buf;- for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {- struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));- if (!cur) {- // some tensors may be allocated in a different context- continue;- }-+ for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {if (progress_callback) {if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {return false;@@ -2421,7 +2652,7 @@ struct llama_model_loader {// load LLaMA models//-static std::string llama_model_arch_name(llm_arch arch) {+static const char * llama_model_arch_name(llm_arch arch) {auto it = LLM_ARCH_NAMES.find(arch);if (it == LLM_ARCH_NAMES.end()) {return "unknown";@@ -2456,9 +2687,12 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";- case LLAMA_FTYPE_MOSTLY_IQ2_XXS:return "IQ2_XSS - 2.0625 bpw";+ case LLAMA_FTYPE_MOSTLY_IQ2_XXS:return "IQ2_XXS - 2.0625 bpw";case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";case LLAMA_FTYPE_MOSTLY_Q3_K_XS:return "Q3_K - Extra small";+ case LLAMA_FTYPE_MOSTLY_IQ3_XXS:return "IQ3_XXS - 3.0625 bpw";+ case LLAMA_FTYPE_MOSTLY_IQ1_S :return "IQ1_S - 1.5625 bpw";+ case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";default: return "unknown, may not work";}@@ -2466,12 +2700,20 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {static const char * llama_model_type_name(e_model type) {switch (type) {+ case MODEL_22M: return "22M";+ case MODEL_33M: return "33M";+ case MODEL_109M: return "109M";+ case MODEL_137M: return "137M";+ case MODEL_0_5B: return "0.5B";case MODEL_1B: return "1B";+ case MODEL_2B: return "2B";case MODEL_3B: return "3B";case MODEL_7B: return "7B";case MODEL_8B: return "8B";case MODEL_13B: return "13B";+ case MODEL_14B: return "14B";case MODEL_15B: return "15B";+ case MODEL_20B: return "20B";case MODEL_30B: return "30B";case MODEL_34B: return "34B";case MODEL_40B: return "40B";@@ -2484,6 +2726,15 @@ static const char * llama_model_type_name(e_model type) {default: return "?B";}}+static const char * llama_model_vocab_type_name(enum llama_vocab_type type){+ switch (type) {+ case LLAMA_VOCAB_TYPE_SPM: return "SPM";+ case LLAMA_VOCAB_TYPE_BPE: return "BPE";+ case LLAMA_VOCAB_TYPE_WPM: return "WPM";+ default: return "unknown";+ }+}+static void llm_load_arch(llama_model_loader & ml, llama_model & model) {model.arch = ml.get_arch();@@ -2596,6 +2847,15 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_MINICPM:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);++ switch (hparams.n_layer) {+ case 40: model.type = e_model::MODEL_2B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;case LLM_ARCH_FALCON:{ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);@@ -2614,6 +2874,11 @@ static void llm_load_hparams(case 40: model.type = e_model::MODEL_13B; break;default: model.type = e_model::MODEL_UNKNOWN;}++ if (model.type == e_model::MODEL_13B) {+ // TODO: become GGUF KV parameter+ hparams.f_max_alibi_bias = 8.0f;+ }} break;case LLM_ARCH_STARCODER:{@@ -2641,6 +2906,41 @@ static void llm_load_hparams(case 32: model.type = e_model::MODEL_1B; break;default: model.type = e_model::MODEL_UNKNOWN;}++ // TODO: become GGUF KV parameter+ hparams.f_max_alibi_bias = 8.0f;+ } break;+ case LLM_ARCH_BERT:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);+ ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type);+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type);++ switch (hparams.n_layer) {+ case 3:+ model.type = e_model::MODEL_17M; break; // bge-micro+ case 6:+ model.type = e_model::MODEL_22M; break; // MiniLM-L6+ case 12:+ switch (hparams.n_embd) {+ case 384: model.type = e_model::MODEL_33M; break; // MiniLM-L12, bge-small+ case 768: model.type = e_model::MODEL_109M; break; // bge-base+ } break;+ case 24:+ model.type = e_model::MODEL_335M; break; // bge-large+ }+ } break;+ case LLM_ARCH_NOMIC_BERT:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);+ ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type);+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type);++ if (hparams.n_layer == 12 && hparams.n_embd == 768) {+ model.type = e_model::MODEL_137M;+ }} break;case LLM_ARCH_BLOOM:{@@ -2654,11 +2954,12 @@ static void llm_load_hparams(case 4096: model.type = e_model::MODEL_7B; break;} break;}++ // TODO: become GGUF KV parameter+ hparams.f_max_alibi_bias = 8.0f;} break;case LLM_ARCH_MPT:{- hparams.f_clamp_kqv = 0.0f;-ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv, false);ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);@@ -2738,11 +3039,42 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_ORION:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);++ switch (hparams.n_layer) {+ case 40: model.type = e_model::MODEL_14B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;+ case LLM_ARCH_INTERNLM2:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ switch (hparams.n_layer) {+ case 32: model.type = e_model::MODEL_7B; break;+ case 48: model.type = e_model::MODEL_20B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;+ case LLM_ARCH_GEMMA:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ switch (hparams.n_layer) {+ case 18: model.type = e_model::MODEL_2B; break;+ case 28: model.type = e_model::MODEL_7B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;default: (void)0;}model.ftype = ml.ftype;++ if (hparams.f_max_alibi_bias > 0.0f) {+ hparams.need_kq_pos = true;+ }}// TODO: This should probably be in llama.h@@ -2790,6 +3122,11 @@ static void llm_load_vocab(vocab.special_unk_id = 0;vocab.special_sep_id = -1;vocab.special_pad_id = -1;++ const int add_space_prefix_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_ADD_PREFIX).c_str());+ if (add_space_prefix_keyidx != -1) {+ vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);+ } // The default value of add_space_prefix is true.} else if (tokenizer_name == "gpt2") {vocab.type = LLAMA_VOCAB_TYPE_BPE;@@ -2824,6 +3161,16 @@ static void llm_load_vocab(vocab.special_unk_id = -1;vocab.special_sep_id = -1;vocab.special_pad_id = -1;+ } else if (tokenizer_name == "bert") {+ vocab.type = LLAMA_VOCAB_TYPE_WPM;++ // default special tokens+ vocab.special_bos_id = 101;+ vocab.special_eos_id = 102;+ vocab.special_unk_id = 100;+ vocab.special_sep_id = -1;+ vocab.special_pad_id = -1;+ vocab.add_space_prefix = false;} else {LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_name.c_str());LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__);@@ -2851,7 +3198,14 @@ static void llm_load_vocab(// determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n'if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {- vocab.linefeed_id = llama_byte_to_token(vocab, '\n');+ try {+ vocab.linefeed_id = llama_byte_to_token(vocab, '\n');+ } catch (const std::exception & e) {+ LLAMA_LOG_WARN("%s: SPM vocabulary, but newline token not found: %s! Using special_pad_id instead.", __func__, e.what());+ vocab.linefeed_id = vocab.special_pad_id;+ }+ } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {+ vocab.linefeed_id = vocab.special_pad_id;} else {const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false);GGML_ASSERT(!ids.empty() && "model vocab missing newline token");@@ -2997,12 +3351,12 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {const auto & hparams = model.hparams;const auto & vocab = model.vocab;- const auto rope_scaling_type = LLAMA_ROPE_SCALING_TYPES.at(hparams.rope_scaling_type_train);+ const char * rope_scaling_type = LLAMA_ROPE_SCALING_TYPES.at(hparams.rope_scaling_type_train);// hparamsLLAMA_LOG_INFO("%s: format = %s\n", __func__, llama_file_version_name(ml.fver));- LLAMA_LOG_INFO("%s: arch = %s\n", __func__, LLM_ARCH_NAMES.at(model.arch).c_str());- LLAMA_LOG_INFO("%s: vocab type = %s\n", __func__, vocab.type == LLAMA_VOCAB_TYPE_SPM ? "SPM" : "BPE"); // TODO: fix+ LLAMA_LOG_INFO("%s: arch = %s\n", __func__, LLM_ARCH_NAMES.at(model.arch));+ LLAMA_LOG_INFO("%s: vocab type = %s\n", __func__, llama_model_vocab_type_name(vocab.type));LLAMA_LOG_INFO("%s: n_vocab = %u\n", __func__, hparams.n_vocab);LLAMA_LOG_INFO("%s: n_merges = %u\n", __func__, (int) vocab.bpe_ranks.size());LLAMA_LOG_INFO("%s: n_ctx_train = %u\n", __func__, hparams.n_ctx_train);@@ -3023,7 +3377,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff);LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert);LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used);- LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type.c_str());+ LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type);LLAMA_LOG_INFO("%s: freq_base_train = %.1f\n", __func__, hparams.rope_freq_base_train);LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train);LLAMA_LOG_INFO("%s: n_yarn_orig_ctx = %u\n", __func__, hparams.n_yarn_orig_ctx);@@ -3089,21 +3443,18 @@ static bool llm_load_tensors(model.buft_layer[i] = llama_default_buffer_type_cpu(true);}- if (split_mode == LLAMA_SPLIT_LAYER && llamafile_has_cuda()) {+ if (split_mode == LLAMA_SPLIT_LAYER) {// calculate the split points- int device_count = ggml_backend_cuda_get_device_count();+ int device_count = llama_get_device_count();bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; });- float splits[GGML_CUDA_MAX_DEVICES];+ std::vector<float> splits(device_count);if (all_zero) {// default split, by free memoryfor (int i = 0; i < device_count; ++i) {- size_t total;- size_t free;- ggml_backend_cuda_get_device_memory(i, &total, &free);- splits[i] = free;+ splits[i] = llama_get_device_memory(i);}} else {- std::copy(tensor_split, tensor_split + device_count, splits);+ std::copy(tensor_split, tensor_split + device_count, splits.begin());}// sum and normalize the splits to get the split points@@ -3119,12 +3470,12 @@ static bool llm_load_tensors(// assign the repeating layers to the devices according to the splitsint act_gpu_layers = std::min(n_gpu_layers, (int)n_layer + 1);for (int64_t i = i_gpu_start; i < n_layer; ++i) {- int layer_gpu = std::upper_bound(splits, splits + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits;+ int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits.begin();model.buft_layer[i] = llama_default_buffer_type_offload(layer_gpu);}// assign the output layerif (n_gpu_layers > n_layer) {- int layer_gpu = std::upper_bound(splits, splits + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits;+ int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits.begin();model.buft_output = llama_default_buffer_type_offload(layer_gpu);} else {model.buft_output = llama_default_buffer_type_cpu(true);@@ -3167,7 +3518,7 @@ static bool llm_load_tensors(}// create one context per buffer type- size_t ctx_size = ggml_tensor_overhead()*ml.n_tensors;+ size_t ctx_size = ggml_tensor_overhead()*(ml.n_tensors + 1); // +1 for models where tok_embd is duplicated as outputstd::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;for (auto & it : buft_layer_count) {struct ggml_init_params params = {@@ -3192,6 +3543,7 @@ static bool llm_load_tensors(const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();const int64_t n_embd_gqa = n_embd_v_gqa;const int64_t n_vocab = hparams.n_vocab;+ const int64_t n_vocab_type = hparams.n_vocab_type;const int64_t n_ff = hparams.n_ff;GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);@@ -3208,13 +3560,16 @@ static bool llm_load_tensors(switch (model.arch) {case LLM_ARCH_LLAMA:case LLM_ARCH_REFACT:+ case LLM_ARCH_MINICPM:{model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});// output{model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});- model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ if (model.arch != LLM_ARCH_MINICPM){+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }}for (int i = 0; i < n_layer; ++i) {@@ -3301,6 +3656,7 @@ static bool llm_load_tensors(} else {model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPUml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);}}@@ -3403,11 +3759,63 @@ static bool llm_load_tensors(layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64});}} break;+ case LLM_ARCH_BERT:+ case LLM_ARCH_NOMIC_BERT:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type});+ if (model.arch == LLM_ARCH_BERT) {+ model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train});+ }++ model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd});+ model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd});++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ if (model.arch == LLM_ARCH_BERT) {+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});++ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});++ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});+ } else {+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ }++ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});++ layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd});+ layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd});++ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});++ if (model.arch == LLM_ARCH_BERT) {+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});++ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});+ } else {+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ }++ layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd});+ layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd});+ }+ } break;case LLM_ARCH_BLOOM:{- model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});- model.tok_norm = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd});- model.tok_norm_b = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd});+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd});+ model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd});// output{@@ -3718,58 +4126,150 @@ static bool llm_load_tensors(layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});}} break;- default:- throw std::runtime_error("unknown architecture");- }- }+ case LLM_ARCH_ORION:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);- ml.done_getting_tensors();+ auto & layer = model.layers[i];- ml.init_mapping(true, use_mlock ? &model.mlock_mmap : nullptr);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});- // create the backend buffers- std::vector<std::pair<ggml_context *, ggml_backend_buffer_t>> ctx_bufs;+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- for (auto & it : ctx_map) {- ggml_backend_buffer_type_t buft = it.first;- ggml_context * ctx = it.second;- ggml_backend_buffer_t buf = nullptr;+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});- // only the mmap region containing the tensors in the model is mapped to the backend buffer- // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers- // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size- if (ml.use_mmap && buft == llama_default_buffer_type_cpu(true)) {- size_t first, last;- ml.get_mapping_range(&first, &last, ctx);- buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first);- }- else if (ml.use_mmap && buft == ggml_backend_metal_buffer_type()) {- const size_t max_size = ggml_get_max_tensor_size(ctx);- size_t first, last;- ml.get_mapping_range(&first, &last, ctx);- buf = ggml_backend_metal_buffer_from_ptr((char *) ml.mapping->addr + first, last - first, max_size);- }- else {- buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);- if (buf != nullptr && use_mlock && ggml_backend_buffer_is_host(buf)) {- model.mlock_bufs.emplace_back(new llama_mlock);- auto & mlock_buf = model.mlock_bufs.back();- mlock_buf->init (ggml_backend_buffer_get_base(buf));- mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));- }- }- if (buf == nullptr) {- throw std::runtime_error("failed to allocate buffer");- }- // indicate that this buffer contains weights- // this is used by ggml_backend_sched to improve op scheduling -> ops that use a weight are preferably scheduled to the backend that contains the weight- ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);- model.bufs.push_back(buf);- ctx_bufs.emplace_back(ctx, buf);- }+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ }+ } break;+ case LLM_ARCH_INTERNLM2:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});- // print memory requirements- {+ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ // layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});++ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ }+ } break;+ case LLM_ARCH_GEMMA:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // same as tok_embd, duplicated to allow offloading+ ml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);++ const int64_t n_ff = hparams.n_ff;+ const int64_t n_embd_head_k = hparams.n_embd_head_k;+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();++ for (uint32_t i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * hparams.n_head});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * hparams.n_head, n_embd});++ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ }+ } break;+ default:+ throw std::runtime_error("unknown architecture");+ }+ }++ ml.done_getting_tensors();++ ml.init_mapping(true, use_mlock ? &model.mlock_mmap : nullptr);++ // create the backend buffers+ std::vector<std::pair<ggml_context *, ggml_backend_buffer_t>> ctx_bufs;++ for (auto & it : ctx_map) {+ ggml_backend_buffer_type_t buft = it.first;+ ggml_context * ctx = it.second;+ ggml_backend_buffer_t buf = nullptr;++ // only the mmap region containing the tensors in the model is mapped to the backend buffer+ // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers+ // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size+ if (ml.use_mmap && buft == llama_default_buffer_type_cpu(true)) {+ size_t first, last;+ ml.get_mapping_range(&first, &last, ctx);+ buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first);+ }+ else if (ml.use_mmap && buft == ggml_backend_metal_buffer_type()) {+ const size_t max_size = ggml_get_max_tensor_size(ctx);+ size_t first, last;+ ml.get_mapping_range(&first, &last, ctx);+ buf = ggml_backend_metal_buffer_from_ptr((char *) ml.mapping->addr + first, last - first, max_size);+ }+ else {+ buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);+ if (buf != nullptr && use_mlock && ggml_backend_buffer_is_host(buf)) {+ model.mlock_bufs.emplace_back(new llama_mlock);+ auto & mlock_buf = model.mlock_bufs.back();+ mlock_buf->init (ggml_backend_buffer_get_base(buf));+ mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));+ }+ }+ if (buf == nullptr) {+ throw std::runtime_error("failed to allocate buffer");+ }+ // indicate that this buffer contains weights+ // this is used by ggml_backend_sched to improve op scheduling -> ops that use a weight are preferably scheduled to the backend that contains the weight+ ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);+ model.bufs.push_back(buf);+ ctx_bufs.emplace_back(ctx, buf);+ }++ if (llama_supports_gpu_offload()) {const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer));LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_gpu);@@ -3781,10 +4281,11 @@ static bool llm_load_tensors(const int max_offloadable_layers = hparams.n_layer + 1;LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);+ }- for (ggml_backend_buffer_t buf : model.bufs) {- LLAMA_LOG_INFO("%s: %10s buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0);- }+ // print memory requirements+ for (ggml_backend_buffer_t buf : model.bufs) {+ LLAMA_LOG_INFO("%s: %10s buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0);}// populate tensors_by_name@@ -3812,15 +4313,27 @@ static bool llm_load_tensors(}// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback-static int llama_model_load(const std::string & fname, llama_model & model, const llama_model_params & params) {+static int llama_model_load(const std::string & fname, llama_model & model, llama_model_params & params) {try {llama_model_loader ml(fname, params.use_mmap, params.kv_overrides);model.hparams.vocab_only = params.vocab_only;- llm_load_arch (ml, model);- llm_load_hparams(ml, model);- llm_load_vocab (ml, model);+ try {+ llm_load_arch(ml, model);+ } catch(const std::exception & e) {+ throw std::runtime_error("error loading model architecture: " + std::string(e.what()));+ }+ try {+ llm_load_hparams(ml, model);+ } catch(const std::exception & e) {+ throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what()));+ }+ try {+ llm_load_vocab(ml, model);+ } catch(const std::exception & e) {+ throw std::runtime_error("error loading model vocabulary: " + std::string(e.what()));+ }llm_load_print_meta(ml, model);@@ -3833,6 +4346,22 @@ static int llama_model_load(const std::string & fname, llama_model & model, consreturn 0;}+#ifdef GGML_USE_KOMPUTE+ if (params.n_gpu_layers > 0 && (+ !(model.arch == LLM_ARCH_LLAMA || model.arch == LLM_ARCH_FALCON)+ || !(+ model.ftype == LLAMA_FTYPE_ALL_F32 ||+ model.ftype == LLAMA_FTYPE_MOSTLY_F16 ||+ model.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ||+ model.ftype == LLAMA_FTYPE_MOSTLY_Q4_1+ )+ )) {+ // TODO(cebtenzzre): propagate this error outside of llama_load_model_from_file+ LLAMA_LOG_WARN("%s: disabling Kompute due to unsupported model arch or quantization\n", __func__);+ params.n_gpu_layers = 0;+ }+#endif+if (!llm_load_tensors(ml, model, params.n_gpu_layers, params.split_mode, params.main_gpu, params.tensor_split, params.use_mlock,params.progress_callback, params.progress_callback_user_data@@ -4121,10 +4650,10 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * wo_b,struct ggml_tensor * q_cur,struct ggml_tensor * kq_mask,+ struct ggml_tensor * kq_pos,int64_t n_ctx,int32_t n_tokens,int32_t n_kv,- float max_alibi_bias,float kq_scale,const llm_build_cb & cb,int il) {@@ -4154,26 +4683,26 @@ static struct ggml_tensor * llm_build_kqv(ggml_mul_mat_set_prec(kq, GGML_PREC_F32);}- if (max_alibi_bias > 0.0f) {- // temporary branch until we figure out how to handle ggml_alibi through ggml_add+#if defined(GGML_USE_VULKAN) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_SYCL)+#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Vulkan, Kompute, and SYCL")+#pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024")+#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488")+ if (hparams.f_max_alibi_bias > 0.0f) {kq = ggml_scale(ctx, kq, kq_scale);cb(kq, "kq_scaled", il);- if (max_alibi_bias > 0.0f) {- // TODO: n_head or n_head_kv- // TODO: K-shift is likely not working- // TODO: change to ggml_add- kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, max_alibi_bias);- cb(kq, "kq_scaled_alibi", il);- }+ kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, hparams.f_max_alibi_bias);+ cb(kq, "kq_scaled_alibi", il);kq = ggml_add(ctx, kq, kq_mask);cb(kq, "kq_masked", il);kq = ggml_soft_max(ctx, kq);cb(kq, "kq_soft_max", il);- } else {- kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale);+ } else+#endif+ {+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_pos, kq_scale, hparams.f_max_alibi_bias);cb(kq, "kq_soft_max_ext", il);}@@ -4221,11 +4750,11 @@ static struct ggml_tensor * llm_build_kv(struct ggml_tensor * v_cur,struct ggml_tensor * q_cur,struct ggml_tensor * kq_mask,+ struct ggml_tensor * kq_pos,int64_t n_ctx,int32_t n_tokens,int32_t kv_head,int32_t n_kv,- float max_alibi_bias,float kq_scale,const llm_build_cb & cb,int il) {@@ -4239,9 +4768,8 @@ static struct ggml_tensor * llm_build_kv(llm_build_kv_store(ctx, hparams, kv, graph, k_cur, v_cur, n_ctx, n_tokens, kv_head, cb, il);struct ggml_tensor * cur;- cur = llm_build_kqv(ctx, model, hparams, kv, graph,- wo, wo_b,- q_cur, kq_mask, n_ctx, n_tokens, n_kv, max_alibi_bias, kq_scale, cb, il);+ cur = llm_build_kqv(ctx, model, hparams, kv, graph, wo, wo_b,+ q_cur, kq_mask, kq_pos, n_ctx, n_tokens, n_kv, kq_scale, cb, il);cb(cur, "kqv_out", il);return cur;@@ -4282,6 +4810,7 @@ struct llm_build_context {const int32_t n_orig_ctx;const bool do_rope_shift;+ const uint32_t pooling_type;const llm_build_cb & cb;@@ -4325,6 +4854,7 @@ struct llm_build_context {kv_head (worst_case ? n_ctx - n_tokens : kv_self.head),n_orig_ctx (cparams.n_yarn_orig_ctx),do_rope_shift (worst_case || kv_self.has_shift),+ pooling_type (cparams.do_pooling ? hparams.pooling_type : (uint32_t)LLAMA_POOLING_NONE),cb (cb),buf_compute_meta (lctx.buf_compute_meta) {// all initializations should be done in init()@@ -4407,7 +4937,7 @@ struct llm_build_context {}Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -4422,7 +4952,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4552,6 +5082,10 @@ struct llm_build_context {struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);+ // positions of the tokens in the KV cache+ struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);+ cb(KQ_pos, "KQ_pos", -1);+// shift the entire K-cache if neededif (do_rope_shift) {llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);@@ -4600,12 +5134,9 @@ struct llm_build_context {cb(Kcur, "Kcur", il);- // apply ALiBi for 13B model- const float max_alibi_bias = model.type == MODEL_13B ? 8.0f : -1.0f;-cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4729,7 +5260,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4828,7 +5359,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5033,7 +5564,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Q, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Q, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5095,6 +5626,10 @@ struct llm_build_context {struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);+ // positions of the tokens in the KV cache+ struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);+ cb(KQ_pos, "KQ_pos", -1);+for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -5122,7 +5657,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5168,6 +5703,151 @@ struct llm_build_context {return gf;}+ struct ggml_cgraph * build_bert() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ // get input vectors with right size+ const size_t stride1 = n_tokens * ggml_type_size(lctx.inp_tokens->type);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);+ struct ggml_tensor * inp_mean = ggml_view_2d(ctx0, lctx.inp_mean, n_tokens, n_tokens, stride1, 0);+ struct ggml_tensor * inp_cls = ggml_view_1d(ctx0, lctx.inp_cls, n_tokens, 0);++ // construct input embeddings (token, type, position)+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);++ // token types are hardcoded to zero ("Sentence A")+ struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0);+ inpL = ggml_add(ctx0, inpL, type_row0);+ if (model.arch == LLM_ARCH_BERT) {+ inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL);+ }+ cb(inpL, "inp_embd", -1);++ // embed layer norm+ inpL = llm_build_norm(ctx0, inpL, hparams, model.tok_norm, model.tok_norm_b, LLM_NORM, cb, -1);+ cb(inpL, "inp_norm", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);+ cb(KQ_mask, "KQ_mask", -1); // [n_kv, n_tokens]++ // iterate layers+ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * cur = inpL;++ // self-attention+ if (model.arch == LLM_ARCH_BERT) {+ struct ggml_tensor * Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq);+ cb(Qcur, "Qcur", il);++ struct ggml_tensor * Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk);+ cb(Kcur, "Kcur", il);++ struct ggml_tensor * Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv);+ cb(Vcur, "Vcur", il);++ // seems like we just need to do this for Q?+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ } else {+ // compute Q and K and RoPE them+ cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur);+ cb(cur, "wqkv", il);++ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));++ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ }++ // re-add the layer input+ cur = ggml_add(ctx0, cur, inpL);++ // attention layer norm+ cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il);++ struct ggml_tensor * ffn_inp = cur;+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ if (model.arch == LLM_ARCH_BERT) {+ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, model.layers[il].ffn_up_b,+ NULL, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);+ } else {+ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ }+ cb(cur, "ffn_out", il);++ // attentions bypass the intermediate layer+ cur = ggml_add(ctx0, cur, ffn_inp);++ // output layer norm+ cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, cb, il);++ // input for next layer+ inpL = cur;+ }++ // final output+ cur = inpL;++ // pooling layer+ if (pooling_type == LLAMA_POOLING_MEAN) {+ cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean);+ } else if (pooling_type == LLAMA_POOLING_CLS) {+ cur = ggml_get_rows(ctx0, cur, inp_cls);+ } else {+ GGML_ASSERT(pooling_type == LLAMA_POOLING_NONE && "Invalid pooling type");+ }+ cb(cur, "result_embd", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }+struct ggml_cgraph * build_bloom() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);@@ -5185,6 +5865,10 @@ struct llm_build_context {struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);+ // positions of the tokens in the KV cache+ struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);+ cb(KQ_pos, "KQ_pos", -1);+inpL = llm_build_norm(ctx0, inpL, hparams,model.tok_norm,model.tok_norm_b,@@ -5218,7 +5902,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5278,6 +5962,10 @@ struct llm_build_context {struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);+ // positions of the tokens in the KV cache+ struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);+ cb(KQ_pos, "KQ_pos", -1);+for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * attn_norm;@@ -5311,7 +5999,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, hparams.f_max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5433,7 +6121,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5548,7 +6236,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5669,7 +6357,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5796,7 +6484,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);cb(cur, "kqv_out", il);}@@ -5899,7 +6587,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}struct ggml_tensor * sa_out = cur;@@ -5998,7 +6686,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -6107,7 +6795,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -6149,16 +6837,507 @@ struct llm_build_context {return gf;}++ struct ggml_cgraph * build_orion() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ cb(inpL, "inp_embd", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);+ cb(inp_pos, "inp_pos", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, model.layers[il].attn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ // if (model.layers[il].bq) {+ // Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ // cb(Qcur, "Qcur", il);+ // }++ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ // if (model.layers[il].bk) {+ // Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ // cb(Kcur, "Kcur", il);+ // }++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ // if (model.layers[il].bv) {+ // Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ // cb(Vcur, "Vcur", il);+ // }++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, NULL,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ }++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, model.layers[il].ffn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, model.output_norm_b,+ LLM_NORM, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }++ struct ggml_cgraph * build_internlm2() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ cb(inpL, "inp_embd", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);+ cb(inp_pos, "inp_pos", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ if (model.layers[il].bq) {+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);+ }++ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ if (model.layers[il].bk) {+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);+ }++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ if (model.layers[il].bv) {+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);+ }++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ }++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }++ // ref: https://arxiv.org/abs/2203.03466+ // https://github.com/ggerganov/llama.cpp/issues/5276#issuecomment-1925774738+ // based on the original build_llama() function+ struct ggml_cgraph * build_minicpm() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);++ const int64_t n_embd = hparams.n_embd;+ //TODO: if the model varies, these parameters need to be read from the model+ const int64_t n_embd_base = 256;+ const float scale_embd = 12.0f;+ const float scale_depth = 1.4f;++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ cb(inpL, "inp_embd", -1);++ // scale the input embeddings+ inpL = ggml_scale(ctx0, inpL, scale_embd);+ cb(inpL, "inp_scaled", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);+ cb(inp_pos, "inp_pos", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ if (model.layers[il].bq) {+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);+ }++ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ if (model.layers[il].bk) {+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);+ }++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ if (model.layers[il].bv) {+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);+ }++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ }++ // scale_res - scale the hidden states for residual connection+ const float scale_res = scale_depth/sqrtf(float(n_layer));+ cur = ggml_scale(ctx0, cur, scale_res);+ cb(cur, "hidden_scaled", -1);++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ {+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);+ }++ // scale the hidden states for residual connection+ cur = ggml_scale(ctx0, cur, scale_res);+ cb(cur, "hidden_scaled_ffn", -1);++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head scaling+ const float scale_lmhead = float(n_embd_base)/float(n_embd);+ cur = ggml_scale(ctx0, cur, scale_lmhead);+ cb(cur, "lmhead_scaling", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.tok_embd, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }++ struct ggml_cgraph * build_gemma() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head_k = hparams.n_embd_head_k;++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ cb(inpL, "inp_embd", -1);+ inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));+ cb(inpL, "inp_scaled", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);+ cb(inp_pos, "inp_pos", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);++ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos,+ n_embd_head_k, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ cb(Qcur, "Qcur", il);+ Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k)));+ cb(Qcur, "Qcur_scaled", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos,+ n_embd_head_k, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, NULL,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ cb(cur, "kqv_out", il);+ }+ struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);+ cb(sa_out, "sa_out", il);++ cur = llm_build_norm(ctx0, sa_out, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ // feed-forward network+ {+ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_GELU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);+ }++ cur = ggml_add(ctx0, cur, sa_out);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }};static struct ggml_cgraph * llama_build_graph(llama_context & lctx,- const llama_batch & batch) {+ const llama_batch & batch,+ bool worst_case) {const auto & model = lctx.model;- // check if we should build the worst-case graph (for memory measurement)- const bool worst_case = ggml_tallocr_is_measure(lctx.alloc);-// this callback allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.)llm_build_cb cb = [&](struct ggml_tensor * cur, const char * name, int il) {if (il >= 0) {@@ -6179,67 +7358,6 @@ static struct ggml_cgraph * llama_build_graph(struct llm_build_context llm(lctx, batch, cb, worst_case);- //- // set input data- //-- if (!ggml_tallocr_is_measure(lctx.alloc)) {- if (batch.token) {- const int64_t n_tokens = batch.n_tokens;-- ggml_backend_tensor_set(lctx.inp_tokens, batch.token, 0, n_tokens*ggml_element_size(lctx.inp_tokens));- }-- if (batch.embd) {- const int64_t n_embd = llm.n_embd;- const int64_t n_tokens = batch.n_tokens;-- ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd));- }-- if (batch.pos) {- const int64_t n_tokens = batch.n_tokens;-- ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos));- }-- {- const int64_t n_kv = llm.n_kv;- const int64_t n_tokens = batch.n_tokens;-- GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));- float * data = (float *) lctx.inp_KQ_mask->data;-- for (int h = 0; h < 1; ++h) {- for (int j = 0; j < n_tokens; ++j) {- const llama_pos pos = batch.pos[j];- const llama_seq_id seq_id = batch.seq_id[j][0];-- for (int i = 0; i < n_kv; ++i) {- float f;- if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) {- f = -INFINITY;- } else {- f = 0;- }- data[h*(n_kv*n_tokens) + j*n_kv + i] = f;- }- }- }- }-- if (llm.do_rope_shift) {- const int64_t n_ctx = llm.n_ctx;-- GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer));- int32_t * data = (int32_t *) lctx.inp_K_shift->data;-- for (int i = 0; i < n_ctx; ++i) {- data[i] = lctx.kv_self.cells[i].delta;- }- }- }-llm.init();switch (model.arch) {@@ -6267,6 +7385,11 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_refact();} break;+ case LLM_ARCH_BERT:+ case LLM_ARCH_NOMIC_BERT:+ {+ result = llm.build_bert();+ } break;case LLM_ARCH_BLOOM:{result = llm.build_bloom();@@ -6303,6 +7426,22 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_codeshell();} break;+ case LLM_ARCH_ORION:+ {+ result = llm.build_orion();+ } break;+ case LLM_ARCH_INTERNLM2:+ {+ result = llm.build_internlm2();+ } break;+ case LLM_ARCH_MINICPM:+ {+ result = llm.build_minicpm();+ } break;+ case LLM_ARCH_GEMMA:+ {+ result = llm.build_gemma();+ } break;default:GGML_ASSERT(false);}@@ -6312,6 +7451,129 @@ static struct ggml_cgraph * llama_build_graph(return result;}+static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {+ //+ // set input data+ //++ const auto & hparams = lctx.model.hparams;+ const auto & cparams = lctx.cparams;+ const auto & kv_self = lctx.kv_self;++ if (batch.token) {+ const int64_t n_tokens = batch.n_tokens;++ ggml_backend_tensor_set(lctx.inp_tokens, batch.token, 0, n_tokens*ggml_element_size(lctx.inp_tokens));+ }++ if (batch.embd) {+ const int64_t n_embd = hparams.n_embd;+ const int64_t n_tokens = batch.n_tokens;++ ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd));+ }++ if (batch.pos) {+ const int64_t n_tokens = batch.n_tokens;++ ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos));+ }++ {+ const int64_t n_kv = kv_self.n;+ const int64_t n_tokens = batch.n_tokens;++ assert(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));++ float * data = (float *) lctx.inp_KQ_mask->data;++ for (int h = 0; h < 1; ++h) {+ for (int j = 0; j < n_tokens; ++j) {+ const llama_pos pos = batch.pos[j];+ const llama_seq_id seq_id = batch.seq_id[j][0];++ for (int i = 0; i < n_kv; ++i) {+ float f;+ if (!lctx.kv_self.cells[i].has_seq_id(seq_id) ||+ (hparams.causal_attn && lctx.kv_self.cells[i].pos > pos)) {+ f = -INFINITY;+ } else {+ f = 0;+ }+ data[h*(n_kv*n_tokens) + j*n_kv + i] = f;+ }+ }+ }+ }++ if (hparams.need_kq_pos) {+ const int64_t n_kv = kv_self.n;++ assert(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer));++ float * data = (float *) lctx.inp_KQ_pos->data;++ for (int i = 0; i < n_kv; ++i) {+ data[i] = float(lctx.kv_self.cells[i].pos);+ }+ }++ if (kv_self.has_shift) {+ const int64_t n_ctx = cparams.n_ctx;++ assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer));++ int32_t * data = (int32_t *) lctx.inp_K_shift->data;++ for (int i = 0; i < n_ctx; ++i) {+ data[i] = lctx.kv_self.cells[i].delta;+ }+ }++ if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_MEAN) {+ const int64_t n_tokens = batch.n_tokens;++ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer));+ float * data = (float *) lctx.inp_mean->data;++ memset(lctx.inp_mean->data, 0, n_tokens * n_tokens * ggml_element_size(lctx.inp_mean));++ std::vector<uint64_t> sum(n_tokens, 0);+ for (int i = 0; i < n_tokens; ++i) {+ const llama_seq_id seq_id = batch.seq_id[i][0];+ sum[seq_id] += 1;+ }++ std::vector<float> div(n_tokens, 0.0f);+ for (int i = 0; i < n_tokens; ++i) {+ const uint64_t s = sum[i];+ if (s > 0) {+ div[i] = 1.0f/float(s);+ }+ }++ for (int i = 0; i < n_tokens; ++i) {+ const llama_seq_id seq_id = batch.seq_id[i][0];+ data[seq_id*n_tokens + i] = div[seq_id];+ }+ }++ if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_CLS) {+ const int64_t n_tokens = batch.n_tokens;++ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer));+ uint32_t * data = (uint32_t *) lctx.inp_cls->data;++ for (int i = 0; i < n_tokens; ++i) {+ const llama_seq_id seq_id = batch.seq_id[i][0];+ const llama_pos pos = batch.pos[i];+ if (pos == 0) {+ data[seq_id] = i;+ }+ }+ }+}+// decode a batch of tokens by evaluating the transformer//// - lctx: llama context@@ -6410,17 +7672,22 @@ static int llama_decode_internal(ggml_backend_sched_reset(lctx.sched);ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data);- ggml_cgraph * gf = llama_build_graph(lctx, batch);+ ggml_cgraph * gf = llama_build_graph(lctx, batch, false);// the output is always the last tensor in the graphstruct ggml_tensor * res = gf->nodes[gf->n_nodes - 1];- GGML_ASSERT(strcmp(res->name, "result_output") == 0);-- // the embeddings could be the second to last tensor, or the third to last tensorstruct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2];- if (strcmp(embeddings->name, "result_norm") != 0) {- embeddings = gf->nodes[gf->n_nodes - 3];- GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0);+ if (strcmp(res->name, "result_output") == 0) {+ // the embeddings could be the second to last tensor, or the third to last tensor+ if (strcmp(embeddings->name, "result_norm") != 0) {+ embeddings = gf->nodes[gf->n_nodes - 3];+ GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0);+ }+ } else if (strcmp(res->name, "result_embd") == 0) {+ embeddings = res;+ res = nullptr;+ } else {+ GGML_ASSERT(false);}// LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);@@ -6430,15 +7697,12 @@ static int llama_decode_internal(// TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well// we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering// with the BLAS calls. need a better solution- if (n_tokens >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) {+ // MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is+ // being processed then Accelerate/BLAS will not be involved, so capping would limit performance.+ if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) {n_threads = std::min(4, n_threads);}- const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 1;- if (ggml_cpu_has_cublas() && fully_offloaded) {- n_threads = 1;- }-#ifdef GGML_USE_MPIconst int64_t n_layer = hparams.n_layer;ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer);@@ -6451,6 +7715,9 @@ static int llama_decode_internal(if (lctx.backend_cpu != nullptr) {ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads);}++ llama_set_inputs(lctx, batch);+ggml_backend_sched_graph_compute(lctx.sched, gf);// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));@@ -6490,7 +7757,7 @@ static int llama_decode_internal(// extract logits// TODO: do not compute and extract logits if only embeddings are needed// need to update the graphs to skip "result_output"- {+ if (res) {auto & logits_out = lctx.logits;#ifndef NDEBUG@@ -6534,9 +7801,12 @@ static int llama_decode_internal(if (!lctx.embedding.empty()) {auto & embedding_out = lctx.embedding;- embedding_out.resize(n_embd);+ const int64_t embd_pos = res ? n_embd * (n_tokens-1) : 0;+ const int64_t embd_size = res ? n_embd : n_embd * n_tokens;++ embedding_out.resize(embd_size);ggml_backend_t embeddings_backend = ggml_backend_sched_get_node_backend(lctx.sched, embeddings);- ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), (n_embd*(n_tokens - 1))*sizeof(float), n_embd*sizeof(float));+ ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), embd_pos*sizeof(float), embd_size*sizeof(float));ggml_backend_synchronize(embeddings_backend);}@@ -6600,6 +7870,9 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {GGML_ASSERT(false);return unicode_to_bytes_bpe(token_data.text);}+ case LLAMA_VOCAB_TYPE_WPM: {+ GGML_ASSERT(false);+ }default:GGML_ASSERT(false);}@@ -6610,8 +7883,15 @@ static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {switch (llama_vocab_get_type(vocab)) {case LLAMA_VOCAB_TYPE_SPM: {const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };- return vocab.token_to_id.at(buf);+ auto token = vocab.token_to_id.find(buf);+ if (token != vocab.token_to_id.end()) {+ return (*token).second;+ }+ // Try to fall back to just the byte as a string+ const char buf2[2] = { (char)ch, 0 };+ return vocab.token_to_id.at(buf2);}+ case LLAMA_VOCAB_TYPE_WPM:case LLAMA_VOCAB_TYPE_BPE: {return vocab.token_to_id.at(bytes_to_unicode_bpe(ch));}@@ -6657,7 +7937,7 @@ struct llm_bigram_spm {};struct llm_tokenizer_spm {- llm_tokenizer_spm(const llama_vocab & vocab): vocab(vocab) {}+ llm_tokenizer_spm(const llama_vocab & vocab) : vocab(vocab) {}void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {// split string into utf8 chars@@ -6732,6 +8012,7 @@ private:if (p == rev_merge.end()) {// output any symbols that did not form tokens as bytes.+ output.reserve(output.size() + symbol.n);for (int j = 0; j < (int)symbol.n; ++j) {llama_vocab::id token_id = llama_byte_to_token(vocab, symbol.text[j]);output.push_back(token_id);@@ -7082,29 +8363,230 @@ private:llm_bigram_bpe::queue work_queue;};-typedef enum FRAGMENT_BUFFER_VARIANT_TYPE{+struct llm_tokenizer_wpm {+ llm_tokenizer_wpm(const llama_vocab & vocab): vocab(vocab) {}++ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {+ auto * token_map = &vocab.token_to_id;++ // normalize and split by whitespace+ std::vector<std::string> words = preprocess(text);++ // bos token prepended already++ // find the longest tokens that form the words+ for (const std::string &word : words) {+ // skip empty words+ if (word.size() == 0) {+ continue;+ }++ // prepend phantom space+ std::string word1 = "\xe2\x96\x81" + word;+ int n = word1.size();++ // we're at the start of a new word+ int i = 0;+ bool match_any = false;++ // move through character position in word+ while (i < n) {+ // loop through possible match length+ bool match = false;+ for (int j = n; j > i; j--) {+ auto it = token_map->find(word1.substr(i, j - i));+ if (it != token_map->end()) {+ output.push_back(it->second);+ match = true;+ match_any = true;+ i = j;+ break;+ }+ }++ // must be an unknown character+ if (!match) {+ i++;+ }+ }++ // we didn't find any matches for this word+ if (!match_any) {+ output.push_back(vocab.special_unk_id);+ }+ }++ // append eos token+ output.push_back(vocab.special_eos_id);+ }++ std::vector<std::string> preprocess(const std::string & text) {+ std::string ori_str = normalize(text);+ uint64_t ori_size = ori_str.size();++ // single punct / single symbol / single digit+ // baseline: add whitespace on the left and right of punct and chinese characters+ std::vector<std::string> words;+ std::string new_str = "";+ uint64_t i = 0;+ while (i < ori_size) {+ int utf_char_len = utf8_len(ori_str[i]);+ if ((utf_char_len == 1) && ispunct(ori_str[i])) {+ new_str += " ";+ new_str += ori_str[i];+ new_str += " ";+ i += 1;+ }+ else if ((utf_char_len == 3) && is_chinese_char(ori_str.substr(i, 3))) {+ new_str += " ";+ new_str += ori_str.substr(i, 3);+ new_str += " ";+ i += 3;+ }+ else {+ new_str += ori_str[i];+ i += 1;+ }+ }++ // split by whitespace+ uint64_t l = 0;+ uint64_t r = 0;+ while (r < new_str.size()) {+ // if is whitespace+ if (isspace(new_str[r])) {+ if (r > l) words.push_back(new_str.substr(l, (r - l)));+ l = r + 1;+ r = l;+ }+ else {+ r += 1;+ }+ }+ if (r > l) {+ words.push_back(new_str.substr(l, (r - l)));+ }+ return words;+ }++ std::string normalize(const std::string & text) {+ // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98+ std::string text2 = strip_accents(text);+ for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i])) {+ char c = text2[i];+ if (c >= 'A' && c <= 'Z') {+ text2[i] = c - 'A' + 'a';+ }+ }+ return text2;+ }++ bool is_chinese_char(const std::string & str) {+ int len = str.length();+ unsigned int codepoint = 0;+ int num_bytes = 0;+ int i = 0;+ unsigned char ch = static_cast<unsigned char>(str[i]);+ if (ch <= 0x7f) {+ codepoint = ch;+ num_bytes = 1;+ } else if ((ch >> 5) == 0x06) {+ codepoint = ch & 0x1f;+ num_bytes = 2;+ } else if ((ch >> 4) == 0x0e) {+ codepoint = ch & 0x0f;+ num_bytes = 3;+ } else if ((ch >> 3) == 0x1e) {+ codepoint = ch & 0x07;+ num_bytes = 4;+ }+ for (int j = 1; j < num_bytes; ++j) {+ if (i + j >= len) {+ return false; // incomplete UTF-8 character+ }+ unsigned char next_ch = static_cast<unsigned char>(str[i + j]);+ if ((next_ch >> 6) != 0x02) {+ return false; // invalid trailing byte+ }+ codepoint = (codepoint << 6) | (next_ch & 0x3f);+ }+ if ((codepoint >= 0x4E00 && codepoint <= 0x9FFF) ||+ (codepoint >= 0x3400 && codepoint <= 0x4DBF) ||+ (codepoint >= 0x20000 && codepoint <= 0x2A6DF) ||+ (codepoint >= 0x2A700 && codepoint <= 0x2B73F) ||+ (codepoint >= 0x2B740 && codepoint <= 0x2B81F) ||+ (codepoint >= 0x2B920 && codepoint <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920+ (codepoint >= 0xF900 && codepoint <= 0xFAFF) ||+ (codepoint >= 0x2F800 && codepoint <= 0x2FA1F) ||+ (codepoint >= 0x3000 && codepoint <= 0x303F) ||+ (codepoint >= 0xFF00 && codepoint <= 0xFFEF)) {+ return true; // NOLINT+ }+ return false;+ }++ std::string strip_accents(const std::string & input_string) {+ std::string resultString;+ std::map<std::string, char> accent_map = {+ {"À", 'A'}, {"Á", 'A'}, {"Â", 'A'}, {"Ã", 'A'}, {"Ä", 'A'}, {"Å", 'A'},+ {"à", 'a'}, {"á", 'a'}, {"â", 'a'}, {"ã", 'a'}, {"ä", 'a'}, {"å", 'a'},+ {"È", 'E'}, {"É", 'E'}, {"Ê", 'E'}, {"Ë", 'E'}, {"è", 'e'}, {"é", 'e'},+ {"ê", 'e'}, {"ë", 'e'}, {"Ì", 'I'}, {"Í", 'I'}, {"Î", 'I'}, {"Ï", 'I'},+ {"ì", 'i'}, {"í", 'i'}, {"î", 'i'}, {"ï", 'i'}, {"Ò", 'O'}, {"Ó", 'O'},+ {"Ô", 'O'}, {"Õ", 'O'}, {"Ö", 'O'}, {"ò", 'o'}, {"ó", 'o'}, {"ô", 'o'},+ {"õ", 'o'}, {"ö", 'o'}, {"Ù", 'U'}, {"Ú", 'U'}, {"Û", 'U'}, {"Ü", 'U'},+ {"ù", 'u'}, {"ú", 'u'}, {"û", 'u'}, {"ü", 'u'}, {"Ý", 'Y'}, {"ý", 'y'},+ {"Ç", 'C'}, {"ç", 'c'}, {"Ñ", 'N'}, {"ñ", 'n'},+ };++ for (size_t i = 0; i < input_string.length();) {+ int len = utf8_len(input_string[i]);+ std::string curChar = input_string.substr(i, len);+ auto iter = accent_map.find(curChar);+ if (iter != accent_map.end()) {+ resultString += iter->second;+ } else {+ resultString += curChar;+ }+ i += len;+ }++ return resultString;+ }++ static size_t utf8_len(char src) {+ const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};+ uint8_t highbits = static_cast<uint8_t>(src) >> 4;+ return lookup[highbits];+ }++ const llama_vocab & vocab;+};++typedef enum FRAGMENT_BUFFER_VARIANT_TYPE {FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN,FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT} FRAGMENT_BUFFER_VARIANT_TYPE;-struct fragment_buffer_variant{+struct fragment_buffer_variant {fragment_buffer_variant(llama_vocab::id _token):type(FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN),token(_token),raw_text(_dummy),offset(0),- length(0){}+ length(0) {}+fragment_buffer_variant(const std::string & _raw_text, int64_t _offset, int64_t _length):type(FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT),- token((llama_vocab::id)-1),+ token((llama_vocab::id) - 1),raw_text(_raw_text),offset(_offset),length(_length){- GGML_ASSERT( _offset >= 0 );- GGML_ASSERT( _length >= 1 );- GGML_ASSERT( offset + length <= raw_text.length() );+ GGML_ASSERT(_offset >= 0);+ GGML_ASSERT(_length >= 1);+ GGML_ASSERT(offset + length <= raw_text.length());}const FRAGMENT_BUFFER_VARIANT_TYPE type;@@ -7117,8 +8599,7 @@ struct fragment_buffer_variant{// #define PRETOKENIZERDEBUG-static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<fragment_buffer_variant> & buffer)-{+static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<fragment_buffer_variant> & buffer) {// for each special tokenfor (const auto & st: vocab.special_tokens_cache) {const auto & special_token = st.first;@@ -7229,17 +8710,15 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &}std::forward_list<fragment_buffer_variant> fragment_buffer;- fragment_buffer.emplace_front( raw_text, 0, raw_text.length() );+ fragment_buffer.emplace_front(raw_text, 0, raw_text.length());- if (special) tokenizer_st_partition( vocab, fragment_buffer );+ if (special) tokenizer_st_partition(vocab, fragment_buffer);switch (vocab.type) {case LLAMA_VOCAB_TYPE_SPM:{- for (const auto & fragment: fragment_buffer)- {- if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT)- {+ for (const auto & fragment : fragment_buffer) {+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {// without adding this leading whitespace, we do not get the same results as the original tokenizer// TODO: It's likely possible to get rid of this string copy entirely@@ -7248,7 +8727,9 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &//auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);if (&fragment == &fragment_buffer.front()) {- raw_text = " " + raw_text; // prefix with space if the first token is not special+ if (vocab.add_space_prefix) {+ raw_text = " " + raw_text; // prefix with space if the first token is not special+ }}#ifdef PRETOKENIZERDEBUG@@ -7257,19 +8738,15 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &llm_tokenizer_spm tokenizer(vocab);llama_escape_whitespace(raw_text);tokenizer.tokenize(raw_text, output);- }- else // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)- {+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)output.push_back(fragment.token);}}} break;case LLAMA_VOCAB_TYPE_BPE:{- for (const auto & fragment: fragment_buffer)- {- if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT)- {+ for (const auto & fragment : fragment_buffer) {+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);#ifdef PRETOKENIZERDEBUG@@ -7277,9 +8754,23 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &#endifllm_tokenizer_bpe tokenizer(vocab);tokenizer.tokenize(raw_text, output);+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)+ output.push_back(fragment.token);}- else // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)- {+ }+ } break;+ case LLAMA_VOCAB_TYPE_WPM:+ {+ for (const auto & fragment : fragment_buffer) {+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);++#ifdef PRETOKENIZERDEBUG+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());+#endif+ llm_tokenizer_wpm tokenizer(vocab);+ tokenizer.tokenize(raw_text, output);+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)output.push_back(fragment.token);}}@@ -7692,7 +9183,7 @@ struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar)void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) {if (seed == LLAMA_DEFAULT_SEED) {- seed = _rand64();+ seed = time(NULL);}ctx->rng.seed(seed);}@@ -7727,8 +9218,17 @@ void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * c}void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int32_t k, size_t min_keep) {+ // TODO: move bucket sort to separate function so that top_p/tail_free/typical/softmax first is equally fast+ // if (k >= (int32_t)candidates->size) {+ // return;+ // }+const int64_t t_start_sample_us = ggml_time_us();+ if (k <= 0) {+ k = candidates->size;+ }+k = std::max(k, (int) min_keep);k = std::min(k, (int) candidates->size);@@ -7835,21 +9335,56 @@ void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * canreturn;}- llama_sample_softmax(ctx, candidates);-const int64_t t_start_sample_us = ggml_time_us();- float scale = candidates->data[0].p; // scale by max prob- size_t i = 1; // first token always matches+ bool min_p_applied = false;++ // if the candidates aren't sorted, try the unsorted implementation first+ if (!candidates->sorted) {+ std::vector<llama_token_data> filtered_tokens;++ float max_logit = -FLT_MAX;+ for (size_t i = 0; i < candidates->size; ++i) {+ max_logit = std::max(max_logit, candidates->data[i].logit);+ }+ const float min_logit = max_logit + logf(p); // min logit for p_i >= p * p_max++ for (size_t i = 0; i < candidates->size; ++i) {+ if (candidates->data[i].logit >= min_logit) {+ filtered_tokens.push_back(candidates->data[i]);+ }+ }- for (; i < candidates->size; ++i) {- if (candidates->data[i].p < p * scale && i >= min_keep) {- break; // prob too small+ // if we have enough values the operation was a success+ if (filtered_tokens.size() >= min_keep) {+ memcpy(candidates->data, filtered_tokens.data(), filtered_tokens.size()*sizeof(llama_token_data));+ candidates->size = filtered_tokens.size();+ min_p_applied = true;}}- // Resize the output vector to keep only the matching tokens- candidates->size = i;+ // if the candidates are sorted or the unsorted implementation failed, use this implementation+ if (!min_p_applied) {+ // Sort the logits in descending order+ if (!candidates->sorted) {+ std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {+ return a.logit > b.logit;+ });+ candidates->sorted = true;+ }++ const float min_logit = candidates->data[0].logit + logf(p); // min logit for p_i >= p * p_max+ size_t i = 1; // first token always matches++ for (; i < candidates->size; ++i) {+ if (candidates->data[i].logit < min_logit && i >= min_keep) {+ break; // prob too small+ }+ }++ // Resize the output vector to keep only the matching tokens+ candidates->size = i;+ }if (ctx) {ctx->t_sample_us += ggml_time_us() - t_start_sample_us;@@ -8747,13 +10282,20 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyif (arch == LLM_ARCH_FALCON || nx % QK_K != 0) {new_type = GGML_TYPE_Q8_0;}- else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) {+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) {new_type = GGML_TYPE_Q5_K;}else if (new_type != GGML_TYPE_Q8_0) {new_type = GGML_TYPE_Q6_K;}- } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) {+ } else if (name == "token_embd.weight") {+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) {+ new_type = GGML_TYPE_Q2_K;+ }+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {+ new_type = GGML_TYPE_Q4_K;+ }+ } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) {if (name.find("attn_v.weight") != std::string::npos) {if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;else new_type = GGML_TYPE_Q2_K;@@ -8763,7 +10305,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyif (qs.i_ffn_down < qs.n_ffn_down/8) new_type = GGML_TYPE_Q2_K;++qs.i_ffn_down;}- else if (name == "token_embd.weight") new_type = GGML_TYPE_Q2_K;+ else if (name.find("attn_output.weight") != std::string::npos) {+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) new_type = GGML_TYPE_IQ2_XXS;+ }} else if (name.find("attn_v.weight") != std::string::npos) {if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;@@ -8771,10 +10315,16 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyelse if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {new_type = GGML_TYPE_Q4_K;}+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {+ new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_Q3_K : GGML_TYPE_IQ3_XXS;+ }else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;}else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL && qs.model.hparams.n_gqa() >= 4) {+ new_type = GGML_TYPE_Q5_K;+ }else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) &&use_more_bits(qs.i_attention_wv, qs.n_attention_wv)) new_type = GGML_TYPE_Q6_K;else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) new_type = GGML_TYPE_Q5_K;@@ -8808,6 +10358,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyelse if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) {if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K;}+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) {+ new_type = i_layer < n_layer/8 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;+ }else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K: arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K@@ -8824,6 +10377,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyif (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;}}+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL && !qs.has_imatrix) {+ if (i_layer < n_layer/8) new_type = GGML_TYPE_Q5_K;+ }else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) {new_type = GGML_TYPE_Q5_K;@@ -8839,13 +10395,14 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty} else if (name.find("attn_output.weight") != std::string::npos) {if (arch != LLM_ARCH_FALCON) {if (qs.model.hparams.n_expert == 8) {- if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS ||- ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ||+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||+ ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL ||ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {new_type = GGML_TYPE_Q5_K;}} else {if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K;+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_Q3_K;else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) new_type = GGML_TYPE_Q4_K;else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;}@@ -8874,6 +10431,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty}++qs.i_ffn_up;}+// if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;//}// IK: let's remove this, else Q2_K is almost the same as Q3_K_S@@ -8888,7 +10446,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tybool convert_incompatible_tensor = false;if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K ||new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K ||- new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS) {+ new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS ||+ new_type == GGML_TYPE_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) {int nx = tensor->ne[0];int ny = tensor->ne[1];if (nx % QK_K != 0) {@@ -8902,8 +10461,10 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyswitch (new_type) {case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:- case GGML_TYPE_Q2_K: new_type = GGML_TYPE_Q4_0; break;- case GGML_TYPE_Q3_K: new_type = GGML_TYPE_Q4_1; break;+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K: new_type = GGML_TYPE_IQ4_NL; break;case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break;case GGML_TYPE_Q5_K: new_type = GGML_TYPE_Q5_1; break;case GGML_TYPE_Q6_K: new_type = GGML_TYPE_Q8_0; break;@@ -8931,18 +10492,21 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s// K-quantscase LLAMA_FTYPE_MOSTLY_Q2_K_S:- case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break;+ case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break;case LLAMA_FTYPE_MOSTLY_Q3_K_XS:case LLAMA_FTYPE_MOSTLY_Q3_K_S:case LLAMA_FTYPE_MOSTLY_Q3_K_M:- case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break;+ case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break;case LLAMA_FTYPE_MOSTLY_Q4_K_S:- case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break;+ case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break;case LLAMA_FTYPE_MOSTLY_Q5_K_S:- case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break;- case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break;- case LLAMA_FTYPE_MOSTLY_IQ2_XXS:quantized_type = GGML_TYPE_IQ2_XXS; break;- case LLAMA_FTYPE_MOSTLY_IQ2_XS :quantized_type = GGML_TYPE_IQ2_XS; break;+ case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break;+ case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_XXS: quantized_type = GGML_TYPE_IQ2_XXS; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_XS: quantized_type = GGML_TYPE_IQ2_XS; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break;+ case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S; break;+ case LLAMA_FTYPE_MOSTLY_IQ4_NL: quantized_type = GGML_TYPE_IQ4_NL; break;default: throw std::runtime_error(format("invalid output file type %d\n", ftype));}@@ -9072,7 +10636,11 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const squantize &= !params->only_copy;// do not quantize expert gating tensors- quantize &= name.find("ffn_gate_inp.weight") == std::string::npos;+ quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_FFN_GATE_INP, "weight");++ // do not quantize positional embeddings and token types (BERT)+ quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight");+ quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_TOKEN_TYPES, "weight");enum ggml_type new_type;void * new_data;@@ -9112,6 +10680,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s}if ((new_type == GGML_TYPE_IQ2_XXS ||new_type == GGML_TYPE_IQ2_XS ||+ new_type == GGML_TYPE_IQ1_S ||(new_type == GGML_TYPE_Q2_K && params->ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && strcmp(tensor->name, "token_embd.weight") != 0)) && !imatrix) {LLAMA_LOG_ERROR("\n\n============================================================\n");LLAMA_LOG_ERROR("Missing importance matrix for tensor %s in a very low-bit quantization\n", tensor->name);@@ -9346,7 +10915,7 @@ static int llama_apply_lora_from_file_internal({LLAMA_LOG_ERROR("%s: invalid tensor data type '%d'\n",__func__, ftype);- return false;+ return 1;}}@@ -9574,6 +11143,7 @@ struct llama_context_params llama_context_default_params() {/*.logits_all =*/ false,/*.embedding =*/ false,/*.offload_kqv =*/ true,+ /*.do_pooling =*/ true,};return result;@@ -9593,19 +11163,43 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {return result;}-int32_t llama_max_devices(void) {- return LLAMA_MAX_DEVICES;+size_t llama_max_devices(void) {+ return GGML_CUDA_MAX_DEVICES;+// #if defined(GGML_USE_METAL)+// return 1;+// #elif defined(GGML_USE_CUBLAS)+// return GGML_CUDA_MAX_DEVICES;+// #elif defined(GGML_USE_SYCL)+// return GGML_SYCL_MAX_DEVICES;+// #elif defined(GGML_USE_VULKAN)+// return GGML_VK_MAX_DEVICES;+// #else+// return 1;+// #endif}-bool llama_mmap_supported(void) {+bool llama_supports_mmap(void) {return llama_mmap::SUPPORTED;}-bool llama_mlock_supported(void) {+bool llama_supports_mlock(void) {return llama_mlock::SUPPORTED;}-void llama_backend_init(bool numa) {+bool llama_supports_gpu_offload(void) {+ return llamafile_has_gpu();+}++// deprecated:+bool llama_mmap_supported(void) {+ return llama_supports_mmap();+}++bool llama_mlock_supported(void) {+ return llama_supports_mlock();+}++void llama_backend_init(void) {ggml_time_init();// needed to initialize f16 tables@@ -9615,15 +11209,17 @@ void llama_backend_init(bool numa) {ggml_free(ctx);}- if (numa) {- ggml_numa_init();- }-#ifdef GGML_USE_MPIggml_mpi_backend_init();#endif}+void llama_numa_init(enum ggml_numa_strategy numa) {+ if (numa != GGML_NUMA_STRATEGY_DISABLED) {+ ggml_numa_init(numa);+ }+}+void llama_backend_free(void) {#ifdef GGML_USE_MPIggml_mpi_backend_free();@@ -9636,8 +11232,8 @@ int64_t llama_time_us(void) {}struct llama_model * llama_load_model_from_file(- const char * path_model,- struct llama_model_params params) {+ const char * path_model,+ struct llama_model_params params) {ggml_time_init();llama_model * model = new llama_model;@@ -9700,6 +11296,7 @@ struct llama_context * llama_new_context_with_model(cparams.yarn_beta_slow = params.yarn_beta_slow;cparams.mul_mat_q = params.mul_mat_q;cparams.offload_kqv = params.offload_kqv;+ cparams.do_pooling = params.do_pooling;cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;@@ -9778,6 +11375,39 @@ if (llamafile_has_metal()) {}}}+// #elif defined(GGML_USE_VULKAN)+// if (model->n_gpu_layers > 0) {+// for (int device = 0; device < ggml_backend_vk_get_device_count(); ++device) {+// ggml_backend_t backend = ggml_backend_vk_init(device);+// if (backend == nullptr) {+// LLAMA_LOG_ERROR("%s: failed to initialize Vulkan%d backend\n", __func__, device);+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// }+// }+// #elif defined(GGML_USE_SYCL)+// if (model->n_gpu_layers > 0) {+// ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu);+// if (backend == nullptr) {+// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d backend\n", __func__, model->main_gpu);+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// }+// #elif defined(GGML_USE_KOMPUTE)+// if (model->n_gpu_layers > 0) {+// auto * backend = ggml_backend_kompute_init(model->main_gpu);+// if (backend == nullptr) {+// LLAMA_LOG_ERROR("%s: failed to initialize Kompute backend\n", __func__);+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// }+// #endif}ctx->backend_cpu = ggml_backend_cpu_init();if (ctx->backend_cpu == nullptr) {@@ -9815,14 +11445,14 @@ if (llamafile_has_metal()) {// resized during inference, reserve maximumctx->logits.reserve(hparams.n_vocab*cparams.n_batch);- if (params.embedding){+ if (params.embedding) {ctx->embedding.resize(hparams.n_embd);}// graph inputs{ggml_init_params init_params = {- /* .mem_size */ ggml_tensor_overhead()*5,+ /* .mem_size */ ggml_tensor_overhead()*8,/* .mem_buffer */ nullptr,/* .no_alloc */ true,};@@ -9832,13 +11462,19 @@ if (llamafile_has_metal()) {ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch);ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch);+ ctx->inp_KQ_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx);ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx);+ ctx->inp_mean = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch);+ ctx->inp_cls = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);ggml_set_name(ctx->inp_tokens, "inp_tokens");ggml_set_name(ctx->inp_embd, "inp_embd");ggml_set_name(ctx->inp_pos, "inp_pos");ggml_set_name(ctx->inp_KQ_mask, "inp_KQ_mask");+ ggml_set_name(ctx->inp_KQ_pos, "inp_KQ_pos");ggml_set_name(ctx->inp_K_shift, "inp_K_shift");+ ggml_set_name(ctx->inp_mean, "inp_mean");+ ggml_set_name(ctx->inp_cls, "inp_cls");ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true));@@ -9864,23 +11500,27 @@ if (llamafile_has_metal()) {ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead());ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES);- ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu);// build worst-case graphint n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch);int n_past = cparams.n_ctx - n_tokens;llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph- ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0));+ ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0), true);// initialize scheduler with the worst-case graph- ggml_backend_sched_init_measure(ctx->sched, gf);- ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu);+ if (!ggml_backend_sched_reserve(ctx->sched, gf)) {+ LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__);+ llama_free(ctx);+ return nullptr;+ }- for (ggml_backend_t backend : ctx->backends) {- ggml_backend_buffer_t buf = ggml_backend_sched_get_buffer(ctx->sched, backend);+ for (size_t i = 0; i < ctx->backends.size(); i++) {+ ggml_backend_t backend = ctx->backends[i];+ ggml_backend_buffer_type_t buft = backend_buft[i];+ size_t size = ggml_backend_sched_get_buffer_size(ctx->sched, backend);LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,- ggml_backend_buffer_name(buf),- ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0);+ ggml_backend_buft_name(buft),+ size / 1024.0 / 1024.0);}// note: the number of splits during measure is higher than during inference due to the kv shift@@ -9983,7 +11623,7 @@ int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int3int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {return snprintf(buf, buf_size, "%s %s %s",- llama_model_arch_name(model->arch).c_str(),+ llama_model_arch_name(model->arch),llama_model_type_name(model->type),llama_model_ftype_name(model->ftype).c_str());}@@ -10326,18 +11966,19 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_datdata_ctx->write(&kv_used, sizeof(kv_used));if (kv_buf_size) {- const size_t elt_size = ggml_element_size(kv_self.k_l[0]);-std::vector<uint8_t> tmp_buf;for (int il = 0; il < (int) n_layer; ++il) {- tmp_buf.resize(elt_size*n_embd_k_gqa*kv_head);+ size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head);+ tmp_buf.resize(k_size);ggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size());data_ctx->write(tmp_buf.data(), tmp_buf.size());// v is not contiguous, copy row by row- tmp_buf.resize(elt_size*kv_head);+ size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head);+ size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx);+ tmp_buf.resize(v_row_size);for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) {- ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), ir*elt_size*n_ctx, tmp_buf.size());+ ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), ir*v_row_stride, tmp_buf.size());data_ctx->write(tmp_buf.data(), tmp_buf.size());}}@@ -10439,17 +12080,16 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {if (kv_buf_size) {GGML_ASSERT(kv_self.total_size() == kv_buf_size);- const size_t elt_size = ggml_element_size(kv_self.k_l[0]);-for (int il = 0; il < (int) n_layer; ++il) {- size_t k_size = elt_size*n_embd_k_gqa*kv_head;+ size_t k_size = ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa*kv_head);ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size);inp += k_size;// v is not contiguous, copy row by row- size_t v_row_size = elt_size*kv_head;+ size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head);+ size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx);for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) {- ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*elt_size*n_ctx, v_row_size);+ ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*v_row_stride, v_row_size);inp += v_row_size;}}@@ -10625,22 +12265,24 @@ struct llama_batch llama_batch_get_one(};}-struct llama_batch llama_batch_init(int32_t n_tokens, int32_t embd, int32_t n_seq_max) {+struct llama_batch llama_batch_init(int32_t n_tokens_alloc, int32_t embd, int32_t n_seq_max) {llama_batch batch = { 0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, 0, 0, };if (embd) {- batch.embd = (float *) malloc(sizeof(float) * n_tokens * embd);+ batch.embd = (float *) malloc(sizeof(float) * n_tokens_alloc * embd);} else {- batch.token = (llama_token *) malloc(sizeof(llama_token) * n_tokens);+ batch.token = (llama_token *) malloc(sizeof(llama_token) * n_tokens_alloc);}- batch.pos = (llama_pos *) malloc(sizeof(llama_pos) * n_tokens);- batch.n_seq_id = (int32_t *) malloc(sizeof(int32_t) * n_tokens);- batch.seq_id = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * n_tokens);- for (int i = 0; i < n_tokens; ++i) {+ batch.pos = (llama_pos *) malloc(sizeof(llama_pos) * n_tokens_alloc);+ batch.n_seq_id = (int32_t *) malloc(sizeof(int32_t) * n_tokens_alloc);+ batch.seq_id = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * (n_tokens_alloc + 1));+ for (int i = 0; i < n_tokens_alloc; ++i) {batch.seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max);}- batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens);+ batch.seq_id[n_tokens_alloc] = nullptr;++ batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens_alloc);return batch;}@@ -10651,7 +12293,7 @@ void llama_batch_free(struct llama_batch batch) {if (batch.pos) free(batch.pos);if (batch.n_seq_id) free(batch.n_seq_id);if (batch.seq_id) {- for (int i = 0; i < batch.n_tokens; ++i) {+ for (int i = 0; batch.seq_id[i] != nullptr; ++i) {free(batch.seq_id[i]);}free(batch.seq_id);@@ -10683,6 +12325,10 @@ float * llama_get_embeddings(struct llama_context * ctx) {return ctx->embedding.data();}+float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) {+ return ctx->embedding.data() + i*ctx->model.hparams.n_embd;+}+const char * llama_token_get_text(const struct llama_model * model, llama_token token) {return model->vocab.id_to_token[token].text.c_str();}@@ -10767,6 +12413,7 @@ static std::string llama_decode_text(const std::string & text) {int32_t llama_token_to_piece(const struct llama_model * model, llama_token token, char * buf, int32_t length) {if (0 <= token && token < llama_n_vocab(model)) {switch (llama_vocab_get_type(model->vocab)) {+ case LLAMA_VOCAB_TYPE_WPM:case LLAMA_VOCAB_TYPE_SPM: {// NOTE: we accept all unsupported token types,// suppressing them like CONTROL tokens.@@ -10832,6 +12479,123 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token tokenreturn 0;}+// trim whitespace from the beginning and end of a string+static std::string trim(const std::string & str) {+ size_t start = 0;+ size_t end = str.size();+ while (start < end && isspace(str[start])) {+ start += 1;+ }+ while (end > start && isspace(str[end - 1])) {+ end -= 1;+ }+ return str.substr(start, end - start);+}++// Simple version of "llama_apply_chat_template" that only works with strings+// This function uses heuristic checks to determine commonly used template. It is not a jinja parser.+static int32_t llama_chat_apply_template_internal(+ const std::string & tmpl,+ const std::vector<const llama_chat_message *> & chat,+ std::string & dest, bool add_ass) {+ // Taken from the research: https://github.com/ggerganov/llama.cpp/issues/5527+ std::stringstream ss;+ if (tmpl.find("<|im_start|>") != std::string::npos) {+ // chatml template+ for (auto message : chat) {+ ss << "<|im_start|>" << message->role << "\n" << message->content << "<|im_end|>\n";+ }+ if (add_ass) {+ ss << "<|im_start|>assistant\n";+ }+ } else if (tmpl.find("[INST]") != std::string::npos) {+ // llama2 template and its variants+ // [variant] support system message+ bool support_system_message = tmpl.find("<<SYS>>") != std::string::npos;+ // [variant] space before + after response+ bool space_around_response = tmpl.find("' ' + eos_token") != std::string::npos;+ // [variant] add BOS inside history+ bool add_bos_inside_history = tmpl.find("bos_token + '[INST]") != std::string::npos;+ // [variant] trim spaces from the input message+ bool strip_message = tmpl.find("content.strip()") != std::string::npos;+ // construct the prompt+ bool is_inside_turn = true; // skip BOS at the beginning+ ss << "[INST] ";+ for (auto message : chat) {+ std::string content = strip_message ? trim(message->content) : message->content;+ std::string role(message->role);+ if (!is_inside_turn) {+ is_inside_turn = true;+ ss << (add_bos_inside_history ? "<s>[INST] " : "[INST] ");+ }+ if (role == "system") {+ if (support_system_message) {+ ss << "<<SYS>>\n" << content << "\n<</SYS>>\n\n";+ } else {+ // if the model does not support system message, we still include it in the first message, but without <<SYS>>+ ss << content << "\n";+ }+ } else if (role == "user") {+ ss << content << " [/INST]";+ } else {+ ss << (space_around_response ? " " : "") << content << (space_around_response ? " " : "") << "</s>";+ is_inside_turn = false;+ }+ }+ // llama2 templates seem to not care about "add_generation_prompt"+ } else if (tmpl.find("<|user|>") != std::string::npos) {+ // zephyr template+ for (auto message : chat) {+ ss << "<|" << message->role << "|>" << "\n" << message->content << "<|endoftext|>\n";+ }+ if (add_ass) {+ ss << "<|assistant|>\n";+ }+ } else {+ // template not supported+ return -1;+ }+ dest = ss.str();+ return dest.size();+}++LLAMA_API int32_t llama_chat_apply_template(+ const struct llama_model * model,+ const char * tmpl,+ const struct llama_chat_message * chat,+ size_t n_msg,+ bool add_ass,+ char * buf,+ int32_t length) {+ std::string curr_tmpl(tmpl == nullptr ? "" : tmpl);+ if (tmpl == nullptr) {+ GGML_ASSERT(model != nullptr);+ // load template from model+ std::vector<char> model_template(2048, 0); // longest known template is about 1200 bytes+ std::string template_key = "tokenizer.chat_template";+ int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), model_template.size());+ if (res < 0) {+ // worst case: there is no information about template, we will use chatml by default+ curr_tmpl = "<|im_start|>"; // see llama_chat_apply_template_internal+ } else {+ curr_tmpl = std::string(model_template.data(), model_template.size());+ }+ }+ // format the chat to string+ std::vector<const llama_chat_message *> chat_vec;+ chat_vec.resize(n_msg);+ for (size_t i = 0; i < n_msg; i++) {+ chat_vec[i] = &chat[i];+ }+ std::string formatted_chat;+ int32_t res = llama_chat_apply_template_internal(curr_tmpl, chat_vec, formatted_chat, add_ass);+ if (res < 0) {+ return res;+ }+ strncpy(buf, formatted_chat.c_str(), length);+ return res;+}+struct llama_timings llama_get_timings(struct llama_context * ctx) {struct llama_timings result = {/*.t_start_ms =*/ 1e-3 * ctx->t_start_us,@@ -10890,6 +12654,7 @@ const char * llama_print_system_info(void) {s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | ";s += "SSSE3 = " + std::to_string(ggml_cpu_has_ssse3()) + " | ";s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | ";+ s += "MATMUL_INT8 = " + std::to_string(ggml_cpu_has_matmul_int8()) + " | ";return s.c_str();}diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 7c418b7a..5254b657 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -4,8 +4,8 @@#define LLAMA_H#include "ggml.h"-#include "ggml-cuda.h"-#define LLAMA_MAX_DEVICES GGML_CUDA_MAX_DEVICES+#include "ggml-backend.h"+#include <stddef.h>#include <stdint.h>#include <stdio.h>@@ -43,8 +43,6 @@#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN#define LLAMA_SESSION_VERSION 4-#define LLAMA_SUPPORTS_GPU_OFFLOAD-#ifdef __cplusplusextern "C" {#endif@@ -65,6 +63,7 @@ extern "C" {enum llama_vocab_type {LLAMA_VOCAB_TYPE_SPM = 0, // SentencePieceLLAMA_VOCAB_TYPE_BPE = 1, // Byte Pair Encoding+ LLAMA_VOCAB_TYPE_WPM = 2, // WordPiece};enum llama_token_type {@@ -102,6 +101,9 @@ extern "C" {LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensorsLLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensorsLLAMA_FTYPE_MOSTLY_Q3_K_XS = 22, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensorsLLAMA_FTYPE_GUESSED = 1024, // not specified in the model file};@@ -114,6 +116,12 @@ extern "C" {LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN,};+ enum llama_pooling_type {+ LLAMA_POOLING_NONE = 0,+ LLAMA_POOLING_MEAN = 1,+ LLAMA_POOLING_CLS = 2,+ };+enum llama_split_mode {LLAMA_SPLIT_NONE = 0, // single GPULLAMA_SPLIT_LAYER = 1, // split layers and KV across GPUs@@ -190,7 +198,7 @@ extern "C" {// LLAMA_SPLIT_LAYER: ignoredint32_t main_gpu;- // proportion of the model (layers or rows) to offload to each GPU, size: LLAMA_MAX_DEVICES+ // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()const float * tensor_split;// Called with a progress value between 0.0 and 1.0. Pass NULL to disable.@@ -216,7 +224,7 @@ extern "C" {uint32_t n_batch; // prompt processing maximum batch sizeuint32_t n_threads; // number of threads to use for generationuint32_t n_threads_batch; // number of threads to use for batch processing- int8_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`+ int32_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`// ref: https://github.com/ggerganov/llama.cpp/pull/2054float rope_freq_base; // RoPE base frequency, 0 = from model@@ -238,6 +246,7 @@ extern "C" {bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)bool embedding; // embedding mode onlybool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU+ bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)};// model quantization parameters@@ -299,6 +308,12 @@ extern "C" {int32_t n_eval;};+ // used in chat template+ typedef struct llama_chat_message {+ const char * role;+ const char * content;+ } llama_chat_message;+// Helpers for getting default parametersLLAMA_API struct llama_model_params llama_model_default_params(void);LLAMA_API struct llama_context_params llama_context_default_params(void);@@ -307,7 +322,10 @@ extern "C" {// Initialize the llama + ggml backend// If numa is true, use NUMA optimizations// Call once at the start of the program- LLAMA_API void llama_backend_init(bool numa);+ LLAMA_API void llama_backend_init(void);++ //optional:+ LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);// Call once at the end of the program - currently only used for MPILLAMA_API void llama_backend_free(void);@@ -327,9 +345,14 @@ extern "C" {LLAMA_API int64_t llama_time_us(void);- LLAMA_API int32_t llama_max_devices(void);- LLAMA_API bool llama_mmap_supported (void);- LLAMA_API bool llama_mlock_supported(void);+ LLAMA_API size_t llama_max_devices(void);++ LLAMA_API bool llama_supports_mmap (void);+ LLAMA_API bool llama_supports_mlock (void);+ LLAMA_API bool llama_supports_gpu_offload(void);++ LLAMA_API DEPRECATED(bool llama_mmap_supported (void), "use llama_supports_mmap() instead");+ LLAMA_API DEPRECATED(bool llama_mlock_supported(void), "use llama_supports_mlock() instead");LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx);@@ -625,6 +648,10 @@ extern "C" {// shape: [n_embd] (1-dimensional)LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);+ // Get the embeddings for the ith sequence+ // llama_get_embeddings(ctx) + i*n_embd+ LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);+//// Vocab//@@ -681,6 +708,25 @@ extern "C" {char * buf,int32_t length);+ /// Apply chat template. Inspired by hf apply_chat_template() on python.+ /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model"+ /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template+ /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model’s default chat template will be used instead.+ /// @param chat Pointer to a list of multiple llama_chat_message+ /// @param n_msg Number of llama_chat_message in this chat+ /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.+ /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)+ /// @param length The size of the allocated buffer+ /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.+ LLAMA_API int32_t llama_chat_apply_template(+ const struct llama_model * model,+ const char * tmpl,+ const struct llama_chat_message * chat,+ size_t n_msg,+ bool add_ass,+ char * buf,+ int32_t length);+//// Grammar//diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex c8258cdd..1a71d9f1 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -14,20 +14,36 @@#include <map>#include <regex>#include <stdexcept>-#include <sstream>#include <vector>--#include "clip.h"+#include <sstream>+#include <cinttypes>+#include <limits>#include "llamafile/log.h"-#include "llama.cpp/ggml.h"-#include "llama.cpp/ggml-cuda.h"+#include "llama.cpp/ggml-backend.h"+#include "llama.cpp/llava/clip.h"#include "llama.cpp/ggml-metal.h"-#include "llama.cpp/ggml-alloc.h"--#define STB_IMAGE_IMPLEMENTATION+#include "llama.cpp/ggml-cuda.h"#include "llama.cpp/stb_image.h"+#include "llama.cpp/ggml.h"-#define CLIP_DEBUG+//#define CLIP_DEBUG_FUNCTIONS++// RGB uint8 image+struct clip_image_u8 {+ int nx;+ int ny;++ std::vector<uint8_t> buf;+};++// RGB float32 image (NHWC)+// Memory layout: RGBRGBRGB...+struct clip_image_f32 {+ int nx;+ int ny;++ std::vector<float> buf;+};static std::string format(const char * fmt, ...) {va_list ap;@@ -48,50 +64,56 @@ static std::string format(const char * fmt, ...) {// key constants//-#define KEY_FTYPE "general.file_type"-#define KEY_NAME "general.name"-#define KEY_DESCRIPTION "general.description"-#define KEY_HAS_TEXT_ENC "clip.has_text_encoder"-#define KEY_HAS_VIS_ENC "clip.has_vision_encoder"+#define KEY_FTYPE "general.file_type"+#define KEY_NAME "general.name"+#define KEY_DESCRIPTION "general.description"+#define KEY_HAS_TEXT_ENC "clip.has_text_encoder"+#define KEY_HAS_VIS_ENC "clip.has_vision_encoder"#define KEY_HAS_LLAVA_PROJ "clip.has_llava_projector"-#define KEY_USE_GELU "clip.use_gelu"-#define KEY_N_EMBD "clip.%s.embedding_length"-#define KEY_N_FF "clip.%s.feed_forward_length"-#define KEY_N_BLOCK "clip.%s.block_count"-#define KEY_N_HEAD "clip.%s.attention.head_count"+#define KEY_USE_GELU "clip.use_gelu"+#define KEY_N_EMBD "clip.%s.embedding_length"+#define KEY_N_FF "clip.%s.feed_forward_length"+#define KEY_N_BLOCK "clip.%s.block_count"+#define KEY_N_HEAD "clip.%s.attention.head_count"#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon"-#define KEY_PROJ_DIM "clip.%s.projection_dim"-#define KEY_TOKENS "tokenizer.ggml.tokens"-#define KEY_N_POSITIONS "clip.text.context_length"-#define KEY_IMAGE_SIZE "clip.vision.image_size"-#define KEY_PATCH_SIZE "clip.vision.patch_size"-#define KEY_IMAGE_MEAN "clip.vision.image_mean"-#define KEY_IMAGE_STD "clip.vision.image_std"-#define KEY_PROJ_TYPE "clip.projector_type"+#define KEY_PROJ_DIM "clip.%s.projection_dim"+#define KEY_TOKENS "tokenizer.ggml.tokens"+#define KEY_N_POSITIONS "clip.text.context_length"+#define KEY_IMAGE_SIZE "clip.vision.image_size"+#define KEY_PATCH_SIZE "clip.vision.patch_size"+#define KEY_IMAGE_MEAN "clip.vision.image_mean"+#define KEY_IMAGE_STD "clip.vision.image_std"+#define KEY_PROJ_TYPE "clip.projector_type"++#define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type"+#define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints"+#define KEY_IMAGE_CROP_RESOLUTION "clip.vision.image_crop_resolution"+//// tensor name constants//-#define TN_TOKEN_EMBD "%s.token_embd.weight"-#define TN_POS_EMBD "%s.position_embd.weight"-#define TN_CLASS_EMBD "v.class_embd"-#define TN_PATCH_EMBD "v.patch_embd.weight"-#define TN_ATTN_K "%s.blk.%d.attn_k.%s"-#define TN_ATTN_Q "%s.blk.%d.attn_q.%s"-#define TN_ATTN_V "%s.blk.%d.attn_v.%s"-#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s"-#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s"-#define TN_FFN_UP "%s.blk.%d.ffn_up.%s"-#define TN_LN_1 "%s.blk.%d.ln1.%s"-#define TN_LN_2 "%s.blk.%d.ln2.%s"-#define TN_LN_PRE "%s.pre_ln.%s"-#define TN_LN_POST "%s.post_ln.%s"-#define TN_TEXT_PROJ "text_projection.weight"-#define TN_VIS_PROJ "visual_projection.weight"-#define TN_LLAVA_PROJ "mm.%d.%s"-#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"+#define TN_TOKEN_EMBD "%s.token_embd.weight"+#define TN_POS_EMBD "%s.position_embd.weight"+#define TN_CLASS_EMBD "v.class_embd"+#define TN_PATCH_EMBD "v.patch_embd.weight"+#define TN_ATTN_K "%s.blk.%d.attn_k.%s"+#define TN_ATTN_Q "%s.blk.%d.attn_q.%s"+#define TN_ATTN_V "%s.blk.%d.attn_v.%s"+#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s"+#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s"+#define TN_FFN_UP "%s.blk.%d.ffn_up.%s"+#define TN_LN_1 "%s.blk.%d.ln1.%s"+#define TN_LN_2 "%s.blk.%d.ln2.%s"+#define TN_LN_PRE "%s.pre_ln.%s"+#define TN_LN_POST "%s.post_ln.%s"+#define TN_TEXT_PROJ "text_projection.weight"+#define TN_VIS_PROJ "visual_projection.weight"+#define TN_LLAVA_PROJ "mm.%d.%s"+#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"+#define TN_IMAGE_NEWLINE "model.image_newline"enum projector_type {@@ -102,8 +124,8 @@ enum projector_type {};static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {- { PROJECTOR_TYPE_MLP, "mlp" },- { PROJECTOR_TYPE_LDP, "ldp" },+ { PROJECTOR_TYPE_MLP, "mlp" },+ { PROJECTOR_TYPE_LDP, "ldp" },};@@ -163,7 +185,6 @@ static std::string gguf_data_to_str(enum gguf_type type, const void * data, int}}-static void replace_all(std::string & s, const std::string & search, const std::string & replace) {std::string result;for (size_t pos = 0; ; pos += search.length()) {@@ -215,7 +236,7 @@ static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {}}-static void print_tensor_info(const ggml_tensor* tensor, const char* prefix = "") {+static void print_tensor_info(const ggml_tensor * tensor, const char * prefix = "") {size_t tensor_size = ggml_nbytes(tensor);tinylogf("%s: n_dims = %d, name = %s, tensor_size=%zu, shape:[%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "], type = %s\n",prefix, ggml_n_dims(tensor), tensor->name, tensor_size,@@ -231,31 +252,136 @@ static projector_type clip_projector_type_from_string(const std::string & name)return PROJECTOR_TYPE_UNKNOWN;}-//-// image data-//+#ifdef CLIP_DEBUG_FUNCTIONS+static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) {+ std::ofstream file(filename, std::ios::binary);+ if (!file.is_open()) {+ std::cerr << "Failed to open file for writing: " << filename << std::endl;+ return;+ }-// RGB uint8 image-struct clip_image_u8 {- int nx;- int ny;+ // PPM header: P6 format, width, height, and max color value+ file << "P6\n" << img.nx << " " << img.ny << "\n255\n";- std::vector<uint8_t> buf;-};+ // Write pixel data+ for (size_t i = 0; i < img.buf.size(); i += 3) {+ // PPM expects binary data in RGB format, which matches our image buffer+ file.write(reinterpret_cast<const char*>(&img.buf[i]), 3);+ }-// RGB float32 image (NHWC)-// Memory layout: RGBRGBRGB...-struct clip_image_f32 {- int nx;- int ny;+ file.close();+}++static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) {+ std::ofstream file(filename, std::ios::binary);+ if (!file.is_open()) {+ std::cerr << "Failed to open file for writing: " << filename << std::endl;+ return;+ }++ int fileSize = 54 + 3 * img.nx * img.ny; // File header + info header + pixel data+ int bytesPerPixel = 3;+ int widthInBytes = img.nx * bytesPerPixel;+ int paddingAmount = (4 - (widthInBytes % 4)) % 4;+ int stride = widthInBytes + paddingAmount;++ // Bitmap file header+ unsigned char fileHeader[14] = {+ 'B','M', // Signature+ 0,0,0,0, // Image file size in bytes+ 0,0,0,0, // Reserved+ 54,0,0,0 // Start of pixel array+ };++ // Total file size+ fileSize = 54 + (stride * img.ny);+ fileHeader[2] = (unsigned char)(fileSize);+ fileHeader[3] = (unsigned char)(fileSize >> 8);+ fileHeader[4] = (unsigned char)(fileSize >> 16);+ fileHeader[5] = (unsigned char)(fileSize >> 24);++ // Bitmap information header (BITMAPINFOHEADER)+ unsigned char infoHeader[40] = {+ 40,0,0,0, // Size of this header (40 bytes)+ 0,0,0,0, // Image width+ 0,0,0,0, // Image height+ 1,0, // Number of color planes+ 24,0, // Bits per pixel+ 0,0,0,0, // No compression+ 0,0,0,0, // Image size (can be 0 for no compression)+ 0,0,0,0, // X pixels per meter (not specified)+ 0,0,0,0, // Y pixels per meter (not specified)+ 0,0,0,0, // Total colors (color table not used)+ 0,0,0,0 // Important colors (all are important)+ };++ // Width and height in the information header+ infoHeader[4] = (unsigned char)(img.nx);+ infoHeader[5] = (unsigned char)(img.nx >> 8);+ infoHeader[6] = (unsigned char)(img.nx >> 16);+ infoHeader[7] = (unsigned char)(img.nx >> 24);+ infoHeader[8] = (unsigned char)(img.ny);+ infoHeader[9] = (unsigned char)(img.ny >> 8);+ infoHeader[10] = (unsigned char)(img.ny >> 16);+ infoHeader[11] = (unsigned char)(img.ny >> 24);++ // Write file headers+ file.write(reinterpret_cast<char*>(fileHeader), sizeof(fileHeader));+ file.write(reinterpret_cast<char*>(infoHeader), sizeof(infoHeader));++ // Pixel data+ std::vector<unsigned char> padding(3, 0); // Max padding size to be added to each row+ for (int y = img.ny - 1; y >= 0; --y) { // BMP files are stored bottom-to-top+ for (int x = 0; x < img.nx; ++x) {+ // Each pixel+ size_t pixelIndex = (y * img.nx + x) * 3;+ unsigned char pixel[3] = {+ img.buf[pixelIndex + 2], // BMP stores pixels in BGR format+ img.buf[pixelIndex + 1],+ img.buf[pixelIndex]+ };+ file.write(reinterpret_cast<char*>(pixel), 3);+ }+ // Write padding for the row+ file.write(reinterpret_cast<char*>(padding.data()), paddingAmount);+ }++ file.close();+}++// debug function to convert f32 to u8+static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) {+ dst.nx = src.nx;+ dst.ny = src.ny;+ dst.buf.resize(3 * src.nx * src.ny);+ for (size_t i = 0; i < src.buf.size(); ++i) {+ dst.buf[i] = static_cast<uint8_t>(std::min(std::max(int(src.buf[i] * 255.0f), 0), 255));+ }+}+#endif- std::vector<float> buf;-};//// clip layers//+struct clip_hparams {+ int32_t image_size;+ int32_t patch_size;+ int32_t hidden_size;+ int32_t n_intermediate;+ int32_t projection_dim;+ int32_t n_head;+ int32_t n_layer;++ float eps;++ char mm_patch_merge_type[32] = "flat"; // spatial_unpad or flat (default)++ int32_t image_grid_pinpoints[32];+ int32_t image_crop_resolution;+};+struct clip_layer {// attentionstruct ggml_tensor * k_w;@@ -285,7 +411,7 @@ struct clip_layer {};struct clip_vision_model {- struct clip_vision_hparams hparams;+ struct clip_hparams hparams;// embeddingsstruct ggml_tensor * class_embedding;@@ -308,6 +434,8 @@ struct clip_vision_model {struct ggml_tensor * mm_2_w = NULL;struct ggml_tensor * mm_2_b = NULL;+ struct ggml_tensor * image_newline = NULL;+// Yi type models with mlp+normalization projectionstruct ggml_tensor * mm_1_w = NULL; // Yi type models have 0, 1, 3, 4struct ggml_tensor * mm_1_b = NULL;@@ -362,10 +490,11 @@ struct clip_ctx {std::vector<uint8_t> buf_compute_meta;// memory buffers to evaluate the model- ggml_backend_buffer_t params_buffer = NULL;+ ggml_backend_buffer_t params_buffer = NULL;ggml_backend_buffer_t compute_buffer = NULL;- ggml_backend_t backend = NULL;- ggml_allocr * compute_alloc = NULL;++ ggml_backend_t backend = NULL;+ ggml_gallocr_t compute_alloc = NULL;};static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch * imgs) {@@ -377,18 +506,19 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32const auto & model = ctx->vision_model;const auto & hparams = model.hparams;- const int image_size = hparams.image_size;- const int patch_size = hparams.patch_size;- const int num_patches = ((image_size / patch_size) * (image_size / patch_size));- const int num_positions = num_patches + 1;- const int hidden_size = hparams.hidden_size;- const int n_head = hparams.n_head;- const int d_head = hidden_size / n_head;- const int n_layer = hparams.n_layer;- //const int n_intermediate = hparams.n_intermediate;- //const int projection_dim = hparams.projection_dim;- const float eps = hparams.eps;- int batch_size = imgs->size;+ const int image_size = hparams.image_size;+ const int patch_size = hparams.patch_size;+ const int num_patches = ((image_size / patch_size) * (image_size / patch_size));+ const int num_patches_per_side = image_size / patch_size; GGML_UNUSED(num_patches_per_side);+ const int num_positions = num_patches + 1;+ const int hidden_size = hparams.hidden_size;+ const int n_head = hparams.n_head;+ const int d_head = hidden_size / n_head;+ const int n_layer = hparams.n_layer;+ const float eps = hparams.eps;++ const int batch_size = imgs->size;+if (ctx->has_llava_projector) {GGML_ASSERT(batch_size == 1);}@@ -403,31 +533,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32struct ggml_cgraph * gf = ggml_new_graph(ctx0);struct ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size, image_size, 3, batch_size);- ggml_allocr_alloc(ctx->compute_alloc, inp_raw);-- if (!ggml_allocr_is_measure(ctx->compute_alloc)) {- float * data = (float *)malloc(ggml_nbytes(inp_raw));-- for (size_t i = 0; i < imgs->size; i++) {- const int nx = imgs->data[i].nx;- const int ny = imgs->data[i].ny;- GGML_ASSERT(nx == image_size && ny == image_size);-- const int n = nx * ny;-- for (int b = 0; b < batch_size; b++) {- for (int k = 0; k < 3; k++) {- for (int y = 0; y < ny; y++) {- for (int x = 0; x < nx; x++) {- data[(b * 3 * n) + k * n + y * nx + x] = imgs->data[b].buf[3 * (y * nx + x) + k];- }- }- }- }- }- ggml_backend_tensor_set(inp_raw, data, 0, ggml_nbytes(inp_raw));- free(data);- }+ ggml_set_name(inp_raw, "inp_raw");+ ggml_set_input(inp_raw);struct ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings, inp_raw, patch_size, patch_size, 0, 0, 1, 1);@@ -436,13 +543,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32// concat class_embeddings and patch_embeddingsstruct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);- ggml_allocr_alloc(ctx->compute_alloc, embeddings);- if (!ggml_allocr_is_measure(ctx->compute_alloc)) {- void* zero_mem = malloc(ggml_nbytes(embeddings));- memset(zero_mem, 0, ggml_nbytes(embeddings));- ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings));- free(zero_mem);- }+ ggml_set_name(embeddings, "embeddings");+ ggml_set_input(embeddings);embeddings = ggml_acc(ctx0, embeddings, model.class_embedding,embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], 0);@@ -451,15 +553,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]);struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);- ggml_allocr_alloc(ctx->compute_alloc, positions);- if (!ggml_allocr_is_measure(ctx->compute_alloc)) {- int* positions_data = (int*)malloc(ggml_nbytes(positions));- for (int i = 0; i < num_positions; i++) {- positions_data[i] = i;- }- ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions));- free(positions_data);- }+ ggml_set_name(positions, "positions");+ ggml_set_input(positions);embeddings =ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, model.position_embeddings, positions));@@ -515,9 +610,9 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32KQ = ggml_soft_max_inplace(ctx0, KQ);struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ);KQV = ggml_reshape_4d(ctx0, KQV, d_head, num_positions, n_head, batch_size);- KQV = ggml_cont(ctx0, ggml_permute(ctx0, KQV, 0, 2, 1, 3));+ KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);- cur = ggml_cpy(ctx0, KQV, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size));+ cur = ggml_cont_3d(ctx0, KQV, hidden_size, num_positions, batch_size);}// attention output@@ -558,15 +653,8 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]);struct ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_patches);- ggml_allocr_alloc(ctx->compute_alloc, patches);- if (!ggml_allocr_is_measure(ctx->compute_alloc)) {- int* patches_data = (int*)malloc(ggml_nbytes(patches));- for (int i = 0; i < num_patches; i++) {- patches_data[i] = i + 1;- }- ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches));- free(patches_data);- }+ ggml_set_name(patches, "patches");+ ggml_set_input(patches);// shape [1, 576, 1024]// ne is whcn, ne = [1024, 576, 1, 1]@@ -580,7 +668,6 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);embeddings = ggml_gelu(ctx0, embeddings);-embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);@@ -812,7 +899,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {}// data- size_t buffer_size = 0;+ size_t model_size = 0;{for (int i = 0; i < n_tensors; ++i) {const char * name = gguf_get_tensor_name(ctx, i);@@ -820,16 +907,14 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {enum ggml_type type = gguf_get_tensor_type(ctx, i);struct ggml_tensor * cur = ggml_get_tensor(meta, name);size_t tensor_size = ggml_nbytes(cur);- buffer_size += tensor_size;+ model_size += tensor_size;if (verbosity >= 3) {tinylogf("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",- __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));+ __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));}}}- buffer_size += n_tensors * 128 /* CLIP PADDING */;-clip_ctx * new_clip = new clip_ctx;// update projector type@@ -838,10 +923,10 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {if (idx != -1) {const std::string proj_type = gguf_get_val_str(ctx, idx);new_clip->proj_type = clip_projector_type_from_string(proj_type);- }- else {+ } else {new_clip->proj_type = PROJECTOR_TYPE_MLP;}+if (new_clip->proj_type == PROJECTOR_TYPE_MLP) {if (gguf_find_tensor(ctx, format(TN_LLAVA_PROJ, 3, "weight").c_str()) != -1) {new_clip->proj_type = PROJECTOR_TYPE_MLP_NORM;@@ -882,12 +967,12 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {tinylogf("%s: text_encoder: %d\n", __func__, new_clip->has_text_encoder);tinylogf("%s: vision_encoder: %d\n", __func__, new_clip->has_vision_encoder);tinylogf("%s: llava_projector: %d\n", __func__, new_clip->has_llava_projector);- tinylogf("%s: model size: %.2f MB\n", __func__, buffer_size / 1024.0 / 1024.0);+ tinylogf("%s: model size: %.2f MB\n", __func__, model_size / 1024.0 / 1024.0);tinylogf("%s: metadata size: %.2f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0);}}- tinylogf("%s: params backend buffer size = % 6.2f MB (%i tensors)\n", __func__, buffer_size / (1024.0 * 1024.0), n_tensors);+ tinylogf("%s: params backend buffer size = % 6.2f MB (%i tensors)\n", __func__, model_size / (1024.0 * 1024.0), n_tensors);// load tensors{@@ -921,12 +1006,10 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {}// alloc memory and offload data- new_clip->params_buffer = ggml_backend_alloc_buffer(new_clip->backend, buffer_size);- ggml_allocr* alloc = ggml_allocr_new_from_buffer(new_clip->params_buffer);+ new_clip->params_buffer = ggml_backend_alloc_ctx_tensors(new_clip->ctx_data, new_clip->backend);for (int i = 0; i < n_tensors; ++i) {const char * name = gguf_get_tensor_name(ctx, i);struct ggml_tensor * cur = ggml_get_tensor(new_clip->ctx_data, name);- ggml_allocr_alloc(alloc, cur);const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i);llamafile_seek(fin, offset, SEEK_SET);if (!fin) {@@ -945,7 +1028,6 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);}}- ggml_allocr_free(alloc);llamafile_close(fin);}@@ -963,11 +1045,41 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {hparams.projection_dim = get_u32(ctx, format(KEY_PROJ_DIM, "vision"));hparams.eps = get_f32(ctx, format(KEY_LAYER_NORM_EPS, "vision"));+ try {+ int idx = get_key_idx(ctx, KEY_IMAGE_GRID_PINPOINTS);+ int n = gguf_get_arr_n(ctx, idx);+ const int32_t * pinpoints = (const int32_t *)gguf_get_arr_data(ctx, idx);+ for (int i = 0; i < 32 && i < n && pinpoints[i] != 0; ++i) {+ hparams.image_grid_pinpoints[i] = pinpoints[i];+ }+ if (n < 32)+ hparams.image_grid_pinpoints[n] = 0;+ } catch (std::runtime_error & e) {+ hparams.image_grid_pinpoints[0]=0;+ }++ try {+ int idx = get_key_idx(ctx, KEY_MM_PATCH_MERGE_TYPE);+ strcpy(hparams.mm_patch_merge_type, gguf_get_val_str(ctx, idx));+ } catch (std::runtime_error & e) {+ strcpy(hparams.mm_patch_merge_type, "flat");+ }++ try {+ hparams.image_crop_resolution = get_u32(ctx, KEY_IMAGE_CROP_RESOLUTION); // llava-1.6+ } catch(const std::exception& e) {+ hparams.image_crop_resolution = hparams.image_size;+ }+int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN);int idx_std = get_key_idx(ctx, KEY_IMAGE_STD);++ const float * mean_data = (const float *)gguf_get_arr_data(ctx, idx_mean);+ const float * std_data = (const float *)gguf_get_arr_data(ctx, idx_std);+for (int i = 0; i < 3; ++i) {- new_clip->image_mean[i] = *((const float *)gguf_get_arr_data(ctx, idx_mean));- new_clip->image_std[i] = *((const float *)gguf_get_arr_data(ctx, idx_std));+ new_clip->image_mean[i] = mean_data[i];+ new_clip->image_std[i] = std_data[i];}if (verbosity >= 2) {@@ -979,13 +1091,27 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {tinylogf("v_projection_dim %d\n", hparams.projection_dim);tinylogf("v_n_head %d\n", hparams.n_head);tinylogf("v_n_layer %d\n", hparams.n_layer);+ tinylogf("v_eps %f\n", hparams.eps);+ tinylogf("v_image_mean %f %f %f\n", new_clip->image_mean[0], new_clip->image_mean[1], new_clip->image_mean[2]);+ tinylogf("v_image_std %f %f %f\n", new_clip->image_std[0], new_clip->image_std[1], new_clip->image_std[2]);+ tinylogf("v_image_grid_pinpoints: ");+ for (int i = 0; i < 32 && (hparams.image_grid_pinpoints[i] != 0); ++i) {+ tinylogf("%d ", hparams.image_grid_pinpoints[i]);+ }+ tinylogf("\n");+ tinylogf("v_mm_patch_merge_type: %s\n", hparams.mm_patch_merge_type);+}- vision_model.patch_embeddings = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD);- vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD);- vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v"));- vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight"));- vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias"));+ try {+ vision_model.patch_embeddings = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD);+ vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD);+ vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v"));+ vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight"));+ vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias"));+ } catch(const std::exception& e) {+ tinylogf("%s: failed to load vision model tensors\n", __func__);+ }// LLaVA projectionif (new_clip->proj_type == PROJECTOR_TYPE_MLP || new_clip->proj_type == PROJECTOR_TYPE_MLP_NORM) {@@ -1011,40 +1137,43 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {vision_model.mm_4_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 4, "weight"));vision_model.mm_4_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 4, "bias"));} catch (std::runtime_error & e) { }- }- else if (new_clip->proj_type == PROJECTOR_TYPE_LDP) {+ try {+ vision_model.image_newline = get_tensor(new_clip->ctx_data, TN_IMAGE_NEWLINE);+ // fprintf(stderr, "%s: image_newline tensor (llava-1.6) found\n", __func__);+ } catch (std::runtime_error & e) { }+ } else if (new_clip->proj_type == PROJECTOR_TYPE_LDP) {// MobileVLM projection- vision_model.mm_model_mlp_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "weight"));- vision_model.mm_model_mlp_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "bias"));- vision_model.mm_model_mlp_3_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "weight"));- vision_model.mm_model_mlp_3_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "bias"));- vision_model.mm_model_block_1_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight"));- vision_model.mm_model_block_1_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight"));- vision_model.mm_model_block_1_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias"));+ vision_model.mm_model_mlp_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "weight"));+ vision_model.mm_model_mlp_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "bias"));+ vision_model.mm_model_mlp_3_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "weight"));+ vision_model.mm_model_mlp_3_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "bias"));+ vision_model.mm_model_block_1_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight"));+ vision_model.mm_model_block_1_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight"));+ vision_model.mm_model_block_1_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias"));vision_model.mm_model_block_1_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight"));vision_model.mm_model_block_1_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias"));vision_model.mm_model_block_1_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight"));vision_model.mm_model_block_1_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias"));- vision_model.mm_model_block_1_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight"));- vision_model.mm_model_block_1_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight"));- vision_model.mm_model_block_1_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias"));- vision_model.mm_model_block_2_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight"));- vision_model.mm_model_block_2_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight"));- vision_model.mm_model_block_2_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias"));+ vision_model.mm_model_block_1_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight"));+ vision_model.mm_model_block_1_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight"));+ vision_model.mm_model_block_1_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias"));+ vision_model.mm_model_block_2_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight"));+ vision_model.mm_model_block_2_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight"));+ vision_model.mm_model_block_2_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias"));vision_model.mm_model_block_2_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight"));vision_model.mm_model_block_2_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias"));vision_model.mm_model_block_2_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight"));vision_model.mm_model_block_2_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias"));- vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight"));- vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight"));- vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias"));- }- else {+ vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight"));+ vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight"));+ vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias"));+ } else {std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type];throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));}vision_model.layers.resize(hparams.n_layer);+for (int il = 0; il < hparams.n_layer; ++il) {auto & layer = vision_model.layers[il];layer.k_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_K, "v", il, "weight"));@@ -1073,15 +1202,12 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {// measure mem requirement and allocate{new_clip->buf_compute_meta.resize(GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead() + ggml_graph_overhead());- new_clip->compute_alloc = ggml_allocr_new_measure_from_backend(new_clip->backend);+ new_clip->compute_alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(new_clip->backend));clip_image_f32_batch batch;batch.size = 1;ggml_cgraph * gf = clip_image_build_graph(new_clip, &batch);- size_t compute_memory_buffer_size = ggml_allocr_alloc_graph(new_clip->compute_alloc, gf);- ggml_allocr_free(new_clip->compute_alloc);- new_clip->compute_buffer = ggml_backend_alloc_buffer(new_clip->backend, compute_memory_buffer_size);- new_clip->compute_alloc = ggml_allocr_new_from_buffer(new_clip->compute_buffer);-+ ggml_gallocr_reserve(new_clip->compute_alloc, gf);+ size_t compute_memory_buffer_size = ggml_gallocr_get_buffer_size(new_clip->compute_alloc, 0);tinylogf("%s: compute allocated memory: %.2f MB\n", __func__, compute_memory_buffer_size /1024.0/1024.0);}@@ -1096,8 +1222,20 @@ struct clip_image_f32 * clip_image_f32_init() {return new clip_image_f32();}-void clip_image_u8_free (struct clip_image_u8 * img) { delete img; }+void clip_image_u8_free(struct clip_image_u8 * img) { delete img; }void clip_image_f32_free(struct clip_image_f32 * img) { delete img; }+void clip_image_u8_batch_free(struct clip_image_u8_batch & batch) {+ if (batch.size > 0) {+ delete[] batch.data;+ batch.size = 0;+ }+}+void clip_image_f32_batch_free(struct clip_image_f32_batch & batch) {+ if (batch.size > 0) {+ delete[] batch.data;+ batch.size = 0;+ }+}static void build_clip_img_from_data(const stbi_uc * data, int nx, int ny, clip_image_u8 * img) {img->nx = nx;@@ -1130,24 +1268,252 @@ bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_lengthreturn true;}-// normalize: x = (x - mean) / std-// TODO: implement bicubic interpolation instead of linear.-bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32 * res, const bool pad2square) {+// Linear interpolation between two points+inline float lerp(float s, float e, float t) {+ return s + (e - s) * t;+}+// Bilinear resize function+static void bilinear_resize(const clip_image_u8& src, clip_image_u8& dst, int target_width, int target_height) {+ dst.nx = target_width;+ dst.ny = target_height;+ dst.buf.resize(3 * target_width * target_height);++ float x_ratio = static_cast<float>(src.nx - 1) / target_width;+ float y_ratio = static_cast<float>(src.ny - 1) / target_height;++ for (int y = 0; y < target_height; y++) {+ for (int x = 0; x < target_width; x++) {+ float px = x_ratio * x;+ float py = y_ratio * y;+ int x_floor = static_cast<int>(px);+ int y_floor = static_cast<int>(py);+ float x_lerp = px - x_floor;+ float y_lerp = py - y_floor;++ for (int c = 0; c < 3; c++) {+ float top = lerp(+ static_cast<float>(src.buf[3 * (y_floor * src.nx + x_floor) + c]),+ static_cast<float>(src.buf[3 * (y_floor * src.nx + (x_floor + 1)) + c]),+ x_lerp+ );+ float bottom = lerp(+ static_cast<float>(src.buf[3 * ((y_floor + 1) * src.nx + x_floor) + c]),+ static_cast<float>(src.buf[3 * ((y_floor + 1) * src.nx + (x_floor + 1)) + c]),+ x_lerp+ );+ dst.buf[3 * (y * target_width + x) + c] = static_cast<uint8_t>(lerp(top, bottom, y_lerp));+ }+ }+ }+}++// Normalize image to float32 - careful with pytorch .to(model.device, dtype=torch.float16) - this sometimes reduces precision (32>16>32), sometimes not+static void normalize_image_u8_to_f32(const clip_image_u8* src, clip_image_f32* dst, const float mean[3], const float std[3]) {+ dst->nx = src->nx;+ dst->ny = src->ny;+ dst->buf.resize(src->buf.size());++ for (size_t i = 0; i < src->buf.size(); ++i) {+ int c = i % 3; // rgb+ dst->buf[i] = (static_cast<float>(src->buf[i]) / 255.0f - mean[c]) / std[c];+ }+}++inline float clip(float x, float lower, float upper) {+ return std::max(lower, std::min(x, upper));+}++static bool bicubic_resize(const clip_image_u8 &img, clip_image_u8 &dst, int target_width, int target_height) {+ const int nx = img.nx;+ const int ny = img.ny;++ dst.nx = target_width;+ dst.ny = target_height;+ dst.buf.resize(3 * target_width * target_height);++ float Cc;+ float C[5];+ float d0, d2, d3, a0, a1, a2, a3;+ int i, j, k, jj;+ int x, y;+ float dx, dy;+ float tx, ty;++ tx = (float)nx / (float)target_width;+ ty = (float)ny / (float)target_height;++ // Bicubic interpolation; adapted from ViT.cpp, inspired from :+ // -> https://github.com/yglukhov/bicubic-interpolation-image-processing/blob/master/libimage.c#L36+ // -> https://en.wikipedia.org/wiki/Bicubic_interpolation++ for (i = 0; i < target_height; i++) {+ for (j = 0; j < target_width; j++) {+ x = (int)(tx * j);+ y = (int)(ty * i);++ dx = tx * j - x;+ dy = ty * i - y;++ for (k = 0; k < 3; k++) {+ for (jj = 0; jj <= 3; jj++) {+ d0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x - 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];+ d2 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];+ d3 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 2, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];+ a0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];++ a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3;+ a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2;+ a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3;++ C[jj] = a0 + a1 * dx + a2 * dx * dx + a3 * dx * dx * dx;++ d0 = C[0] - C[1];+ d2 = C[2] - C[1];+ d3 = C[3] - C[1];+ a0 = C[1];+ a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3;+ a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2;+ a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3;+ Cc = a0 + a1 * dy + a2 * dy * dy + a3 * dy * dy * dy;++ const uint8_t Cc2 = std::min(std::max(std::round(Cc), 0.0f), 255.0f);+ dst.buf[(i * target_width + j) * 3 + k] = float(Cc2);+ }+ }+ }+ }++ return true;+}++// llava-1.6 type of resize_and_pad (black)+static void resize_and_pad_image(const clip_image_u8& image, clip_image_u8 &image_output, const std::pair<int, int>& target_resolution) {+ int target_width = target_resolution.first;+ int target_height = target_resolution.second;++ float scale_w = static_cast<float>(target_width) / image.nx;+ float scale_h = static_cast<float>(target_height) / image.ny;++ int new_width, new_height;++ if (scale_w < scale_h) {+ new_width = target_width;+ new_height = std::min(static_cast<int>(std::ceil(image.ny * scale_w)), target_height);+ } else {+ new_height = target_height;+ new_width = std::min(static_cast<int>(std::ceil(image.nx * scale_h)), target_width);+ }++ clip_image_u8 resized_image;+ // bilinear_resize(image, resized_image, new_width, new_height);+ bicubic_resize(image, resized_image, new_width, new_height);++ clip_image_u8 padded_image;+ padded_image.nx = target_width;+ padded_image.ny = target_height;+ padded_image.buf.resize(3 * target_width * target_height, 0); // Initialize with black++ // Calculate padding offsets+ int pad_x = (target_width - new_width) / 2;+ int pad_y = (target_height - new_height) / 2;++ // Copy the resized image into the center of the padded buffer+ for (int y = 0; y < new_height; ++y) {+ for (int x = 0; x < new_width; ++x) {+ for (int c = 0; c < 3; ++c) {+ padded_image.buf[3 * ((y + pad_y) * target_width + (x + pad_x)) + c] = resized_image.buf[3 * (y * new_width + x) + c];+ }+ }+ }+ image_output = std::move(padded_image);+}++/**+ * Selects the best resolution from a list of possible resolutions based on the original size.+ *+ * @param original_size The original size of the image in the format (width, height).+ * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].+ * @return The best fit resolution in the format (width, height).+ */+static std::pair<int, int> select_best_resolution(const std::pair<int, int> & original_size, const std::vector<std::pair<int, int>> & possible_resolutions) {+ int original_width = original_size.first;+ int original_height = original_size.second;+ std::pair<int, int> best_fit;+ int max_effective_resolution = 0;+ int min_wasted_resolution = std::numeric_limits<int>::max();++ for (const auto& resolution : possible_resolutions) {+ int width = resolution.first;+ int height = resolution.second;+ float scale = std::min(static_cast<float>(width) / original_width, static_cast<float>(height) / original_height);+ int downscaled_width = static_cast<int>(original_width * scale);+ int downscaled_height = static_cast<int>(original_height * scale);+ int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height);+ int wasted_resolution = (width * height) - effective_resolution;+ // fprintf(stderr, "resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution);+ if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) {+ max_effective_resolution = effective_resolution;+ min_wasted_resolution = wasted_resolution;+ best_fit = resolution;+ }+ }++ return best_fit;+}++static std::vector<clip_image_u8*> divide_to_patches_u8(const clip_image_u8 & image, int patch_size) {+ std::vector<clip_image_u8*> patches;+ int width = image.nx;+ int height = image.ny;+ for (int i = 0; i < height; i += patch_size) {+ for (int j = 0; j < width; j += patch_size) {+ clip_image_u8 *patch = clip_image_u8_init();+ patch->nx = std::min(patch_size, width - j);+ patch->ny = std::min(patch_size, height - i);+ patch->buf.resize(3 * patch->nx * patch->ny);+ for (int y = 0; y < patch->ny; ++y) {+ for (int x = 0; x < patch->nx; ++x) {+ for (int c = 0; c < 3; ++c) {+ patch->buf[3 * (y * patch->nx + x) + c] = image.buf[3 * ((i + y) * width + (j + x)) + c];+ }+ }+ }+ patches.push_back(patch);+ }+ }+ return patches;+}++// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector+// res_imgs memory is being allocated here, previous allocations will be freed if found+bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs) {+ bool pad_to_square = true;if (!ctx->has_vision_encoder) {fprintf(stderr, "This gguf file seems to have no vision encoder\n");return false;}+ auto & params = ctx->vision_model.hparams;+ // The model config actually contains all we need to decide on how to preprocess, here we automatically switch to the new llava-1.6 preprocessing+ if (strcmp(params.mm_patch_merge_type, "spatial_unpad") == 0) {+ pad_to_square = false;+ }+ // free the previous res_imgs if any set+ if (res_imgs.size > 0) {+ clip_image_f32_batch_free(res_imgs);+ }+ res_imgs.data = nullptr;+ res_imgs.size = 0;// the logic below is to pad the shorter side to the longer side with a background color: rgb(122, 116, 104)// see https://github.com/haotian-liu/LLaVA/blob/e854a2bf85118c504f6f16bf5c3c7c92f8fa8c6b/llava/conversation.py#L113-L156clip_image_u8 * temp = clip_image_u8_init(); // we will keep the input image data here temporarily- if (pad2square && img->nx != img->ny) {+ if (pad_to_square && img->nx != img->ny) {int longer_side = std::max(img->nx, img->ny);temp->nx = longer_side;temp->ny = longer_side;temp->buf.resize(3 * longer_side * longer_side);- const uint8_t bc[3] = {122, 116, 104}; // background color in RGB from LLaVA+ const uint8_t bc[3] = {122, 116, 104}; // background color in RGB from LLaVA (this is the mean rgb color * 255)// fill with background colorfor (size_t i = 0; i < temp->buf.size(); i++) {@@ -1165,18 +1531,63 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli}}} else {- temp->nx = img->nx;- temp->ny = img->ny;- temp->buf.resize(img->buf.size());- memcpy(temp->buf.data(), img->buf.data(), temp->buf.size());+ if (params.image_grid_pinpoints[0] != 0) {+ // "spatial_unpad" with "anyres" processing for llava-1.6+ std::vector<std::pair<int, int>> possible_resolutions;+ for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) {+ possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]});+ }+ std::pair<int, int> best_resolution = select_best_resolution({img->nx, img->ny}, possible_resolutions);+ // clip_image_save_to_bmp(*img, "input.bmp");+ resize_and_pad_image(*img, *temp, best_resolution); // we do not pad with mean-bg color anymore in llava-1.6+ // clip_image_save_to_bmp(*temp, "resized.bmp");+ // visually verify normalized image:+ // normalize_image_u8_to_f32(*temp, *res, ctx->image_mean, ctx->image_std);+ // {+ // clip_image_u8 * temp2 = clip_image_u8_init();+ // clip_image_convert_f32_to_u8(*res, *temp2);+ // clip_image_save_to_bmp(*temp2, "resized_normalized_f32.bmp");+ // clip_image_u8_free(temp2);+ // }++ std::vector<clip_image_u8 *> patches = divide_to_patches_u8(*temp, params.image_size); // prepare spatial sorted main patches of image_size each (336 in llava-1.6)++ clip_image_u8 *image_original_resize = clip_image_u8_init();+ // bilinear_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are square+ bicubic_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are square+ patches.insert(patches.begin(), image_original_resize);+ // clip_image_f32_batch_init(patches.size());+ res_imgs.size = patches.size();+ res_imgs.data = new clip_image_f32[res_imgs.size];+ int num=0;+ for (auto& patch : patches) {+ normalize_image_u8_to_f32(patch, &res_imgs.data[num], ctx->image_mean, ctx->image_std);+ num++;+ }++ for (size_t i = 0; i < patches.size(); i++) {+ // printf("patch %d: %d %d\n", i, patches[i]->nx, patches[i]->ny);+ clip_image_u8_free(patches[i]);+ }++ clip_image_u8_free(temp);++ return true;+ } else {+ temp->nx = img->nx;+ temp->ny = img->ny;+ temp->buf.resize(img->buf.size());+ memcpy(temp->buf.data(), img->buf.data(), temp->buf.size());+ }}const int nx = temp->nx;const int ny = temp->ny;+ // clip_image_save_to_bmp(*temp, "resized_vanilla.bmp");const int nx2 = ctx->vision_model.hparams.image_size;const int ny2 = ctx->vision_model.hparams.image_size;-+ clip_image_f32 * res = clip_image_f32_init();res->nx = nx2;res->ny = ny2;res->buf.resize(3 * nx2 * ny2);@@ -1230,9 +1641,26 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli}clip_image_u8_free(temp);+ // {+ // clip_image_u8 * temp2 = clip_image_u8_init();+ // clip_image_convert_f32_to_u8(*res, *temp2);+ // clip_image_save_to_bmp(*temp2, "resized_normalized_f32_vanilla.bmp");+ // clip_image_u8_free(temp2);+ // }+ // res_imgs.push_back(res);++ res_imgs.size = 1;+ res_imgs.data = new clip_image_f32[res_imgs.size];+ res_imgs.data[0] = *res;+ clip_image_f32_free(res);+return true;}+ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx) {+ return ctx->vision_model.image_newline;+}+void clip_free(clip_ctx * ctx) {ggml_free(ctx->ctx_data);gguf_free(ctx->ctx_gguf);@@ -1240,6 +1668,42 @@ void clip_free(clip_ctx * ctx) {delete ctx;}+size_t clip_embd_nbytes(const struct clip_ctx * ctx) {+ return clip_n_patches(ctx) * clip_n_mmproj_embd(ctx) * sizeof(float);+}++int32_t clip_image_size(const struct clip_ctx * ctx) {+ return ctx->vision_model.hparams.image_size;+}++int32_t clip_patch_size(const struct clip_ctx * ctx) {+ return ctx->vision_model.hparams.patch_size;+}++int32_t clip_hidden_size(const struct clip_ctx * ctx) {+ return ctx->vision_model.hparams.hidden_size;+}++const char * clip_patch_merge_type(const struct clip_ctx * ctx) {+ return ctx->vision_model.hparams.mm_patch_merge_type;+}++const int32_t * clip_image_grid(const struct clip_ctx * ctx) {+ return ctx->vision_model.hparams.image_grid_pinpoints;+}++int clip_n_patches(const struct clip_ctx * ctx) {+ const auto & params = ctx->vision_model.hparams;++ int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size);++ if (ctx->proj_type == PROJECTOR_TYPE_LDP) {+ n_patches /= 4;+ }++ return n_patches;+}+bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec) {if (!ctx->has_vision_encoder) {fprintf(stderr, "This gguf file seems to have no vision encoder\n");@@ -1259,16 +1723,77 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima}int batch_size = imgs->size;- if(ctx->has_llava_projector) {+ if (ctx->has_llava_projector) {GGML_ASSERT(batch_size == 1); // TODO: support multiple images}- // reset alloc buffer to clean the memory from previous invocations- ggml_allocr_reset(ctx->compute_alloc);-// build the inference graphggml_cgraph * gf = clip_image_build_graph(ctx, imgs);- ggml_allocr_alloc_graph(ctx->compute_alloc, gf);+ ggml_gallocr_alloc_graph(ctx->compute_alloc, gf);++ // set inputs+ const auto & model = ctx->vision_model;+ const auto & hparams = model.hparams;++ const int image_size = hparams.image_size;+ const int patch_size = hparams.patch_size;+ const int num_patches = ((image_size / patch_size) * (image_size / patch_size));+ const int num_positions = num_patches + 1;++ {+ struct ggml_tensor * inp_raw = ggml_graph_get_tensor(gf, "inp_raw");+ float * data = (float *)malloc(ggml_nbytes(inp_raw));++ for (size_t i = 0; i < imgs->size; i++) {+ const int nx = imgs->data[i].nx;+ const int ny = imgs->data[i].ny;+ GGML_ASSERT(nx == image_size && ny == image_size);++ const int n = nx * ny;++ for (int b = 0; b < batch_size; b++) {+ for (int k = 0; k < 3; k++) {+ for (int y = 0; y < ny; y++) {+ for (int x = 0; x < nx; x++) {+ data[(b * 3 * n) + k * n + y * nx + x] = imgs->data[b].buf[3 * (y * nx + x) + k];+ }+ }+ }+ }+ }+ ggml_backend_tensor_set(inp_raw, data, 0, ggml_nbytes(inp_raw));+ free(data);+ }++ {+ struct ggml_tensor * embeddings = ggml_graph_get_tensor(gf, "embeddings");++ void* zero_mem = malloc(ggml_nbytes(embeddings));+ memset(zero_mem, 0, ggml_nbytes(embeddings));+ ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings));+ free(zero_mem);+ }++ {+ struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions");++ int* positions_data = (int*)malloc(ggml_nbytes(positions));+ for (int i = 0; i < num_positions; i++) {+ positions_data[i] = i;+ }+ ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions));+ free(positions_data);+ }++ {+ struct ggml_tensor * patches = ggml_graph_get_tensor(gf, "patches");+ int* patches_data = (int*)malloc(ggml_nbytes(patches));+ for (int i = 0; i < num_patches; i++) {+ patches_data[i] = i + 1;+ }+ ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches));+ free(patches_data);+ }if (ggml_backend_is_cpu(ctx->backend)) {ggml_backend_cpu_set_n_threads(ctx->backend, n_threads);@@ -1285,11 +1810,11 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima// copy the embeddings to the location passed by the userggml_backend_tensor_get(embeddings, vec, 0, ggml_nbytes(embeddings));+return true;}bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) {-ggml_type type = GGML_TYPE_Q4_1;assert(itype < GGML_TYPE_COUNT);@@ -1478,26 +2003,13 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {if (ctx->proj_type == PROJECTOR_TYPE_LDP) {return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0];}- else if (ctx->proj_type == PROJECTOR_TYPE_MLP) {+ if (ctx->proj_type == PROJECTOR_TYPE_MLP) {return ctx->vision_model.mm_2_b->ne[0];- } else if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {- return ctx->vision_model.mm_3_b->ne[0];}- else {- std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type];- throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));- }-}--int clip_n_patches(const struct clip_ctx * ctx) {- auto & params = ctx->vision_model.hparams;- int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size);- if (ctx->proj_type == PROJECTOR_TYPE_LDP) {- n_patches /= 4;+ if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {+ return ctx->vision_model.mm_3_b->ne[0];}- return n_patches;-}-size_t clip_embd_nbytes(const struct clip_ctx * ctx) {- return clip_n_patches(ctx) * clip_n_mmproj_embd(ctx) * sizeof(float);+ std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type];+ throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));}diff --git a/llama.cpp/llava/clip.h b/llama.cpp/llava/clip.hindex 458a256a..e5bd5492 100644--- a/llama.cpp/llava/clip.h+++ b/llama.cpp/llava/clip.h@@ -24,25 +24,7 @@ struct clip_ctx;extern "C" {#endif-struct clip_vision_hparams {- int32_t image_size;- int32_t patch_size;- int32_t hidden_size;- int32_t n_intermediate;- int32_t projection_dim;- int32_t n_head;- int32_t n_layer;- float eps;-};--CLIP_API struct clip_ctx * clip_model_load(const char * fname, int verbosity);--CLIP_API void clip_free(struct clip_ctx * ctx);--CLIP_API size_t clip_embd_nbytes(const struct clip_ctx * ctx);--CLIP_API int clip_n_patches (const struct clip_ctx * ctx);-CLIP_API int clip_n_mmproj_embd(const struct clip_ctx * ctx);+struct clip_ctx;struct clip_image_u8_batch {struct clip_image_u8 * data;@@ -54,18 +36,43 @@ struct clip_image_f32_batch {size_t size;};+CLIP_API struct clip_ctx * clip_model_load (const char * fname, int verbosity);+CLIP_API struct clip_ctx * clip_model_load_cpu(const char * fname, int verbosity);++CLIP_API void clip_free(struct clip_ctx * ctx);++CLIP_API size_t clip_embd_nbytes(const struct clip_ctx * ctx);++CLIP_API int32_t clip_image_size (const struct clip_ctx * ctx);+CLIP_API int32_t clip_patch_size (const struct clip_ctx * ctx);+CLIP_API int32_t clip_hidden_size(const struct clip_ctx * ctx);++// TODO: should be enum, not string+CLIP_API const char * clip_patch_merge_type(const struct clip_ctx * ctx);++CLIP_API const int32_t * clip_image_grid(const struct clip_ctx * ctx);++CLIP_API int clip_n_patches (const struct clip_ctx * ctx);+CLIP_API int clip_n_mmproj_embd(const struct clip_ctx * ctx);+CLIP_API struct clip_image_u8 * clip_image_u8_init ();CLIP_API struct clip_image_f32 * clip_image_f32_init();-CLIP_API void clip_image_u8_free (struct clip_image_u8 * img);+CLIP_API void clip_image_u8_free (struct clip_image_u8 * img);CLIP_API void clip_image_f32_free(struct clip_image_f32 * img);+CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch & batch);+CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch & batch);CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);/** interpret bytes as an image file with length bytes_length, and use the result to populate img */CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img);-CLIP_API bool clip_image_preprocess (struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res, bool pad2square);+/** preprocess img and store the result in res_imgs, pad_to_square may be overriden to false depending on model configuration */+CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs );++CLIP_API struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);+CLIP_API bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cppindex f15c4aa1..bb8a2478 100644--- a/llama.cpp/llava/llava-cli.cpp+++ b/llama.cpp/llava/llava-cli.cpp@@ -39,7 +39,7 @@ static bool eval_id(struct llama_context * ctx_llama, int id, int * n_past) {static bool eval_string(struct llama_context * ctx_llama, const char* str, int n_batch, int * n_past, bool add_bos){std::string str2 = str;- std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos);+ std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos, true);eval_tokens(ctx_llama, embd_inp, n_batch, n_past);return true;}@@ -167,26 +167,32 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_size_t image_pos = prompt.find("<image>");if (image_pos != std::string::npos) {// new templating mode: Provide the full prompt including system message and use <image> as a placeholder for the image-system_prompt = prompt.substr(0, image_pos);user_prompt = prompt.substr(image_pos + std::string("<image>").length());- // We replace \n with actual newlines in user_prompt, just in case -e was not used in templating string- size_t pos = 0;- while ((pos = user_prompt.find("\\n", pos)) != std::string::npos) {- user_prompt.replace(pos, 2, "\n");- pos += 1; // Advance past the replaced newline- }- while ((pos = system_prompt.find("\\n", pos)) != std::string::npos) {- system_prompt.replace(pos, 2, "\n");- pos += 1; // Advance past the replaced newline- }-printf("system_prompt: %s\n", system_prompt.c_str());+ if (params->verbose_prompt) {+ auto tmp = ::llama_tokenize(ctx_llava->ctx_llama, system_prompt, true, true);+ for (int i = 0; i < (int) tmp.size(); i++) {+ printf("%6d -> '%s'\n", tmp[i], llama_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str());+ }+ }printf("user_prompt: %s\n", user_prompt.c_str());+ if (params->verbose_prompt) {+ auto tmp = ::llama_tokenize(ctx_llava->ctx_llama, user_prompt, true, true);+ for (int i = 0; i < (int) tmp.size(); i++) {+ printf("%6d -> '%s'\n", tmp[i], llama_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str());+ }+ }} else {// llava-1.5 native modesystem_prompt = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:";user_prompt = prompt + "\nASSISTANT:";+ if (params->verbose_prompt) {+ auto tmp = ::llama_tokenize(ctx_llava->ctx_llama, user_prompt, true, true);+ for (int i = 0; i < (int) tmp.size(); i++) {+ printf("%6d -> '%s'\n", tmp[i], llama_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str());+ }+ }}eval_string(ctx_llava->ctx_llama, system_prompt.c_str(), params->n_batch, &n_past, add_bos);@@ -198,13 +204,17 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_tinylogf("\n");struct llama_sampling_context * ctx_sampling = llama_sampling_init(params->sparams);-+ std::string response = "";for (int i = 0; i < max_tgt_len; i++) {const char * tmp = sample(ctx_sampling, ctx_llava->ctx_llama, &n_past);+ response += tmp;if (strcmp(tmp, "</s>") == 0) break;if (strstr(tmp, "###")) break; // Yi-VL behavior-printf("%s", tmp);+ if (strstr(response.c_str(), "<|im_end|>")) break; // Yi-34B llava-1.6 - for some reason those decode not as the correct token (tokenizer works)+ if (strstr(response.c_str(), "<|im_start|>")) break; // Yi-34B llava-1.6+ if (strstr(response.c_str(), "USER:")) break; // mistral llava-1.6+fflush(stdout);}@@ -223,7 +233,8 @@ static struct llava_context * llava_init(gpt_params * params) {auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);- llama_backend_init(params->numa);+ llama_backend_init();+ llama_numa_init(params->numa);llama_model_params model_params = llama_model_params_from_gpt_params(*params);diff --git a/llama.cpp/llava/llava.cpp b/llama.cpp/llava/llava.cppindex 2b71f1b6..4400e599 100644--- a/llama.cpp/llava/llava.cpp+++ b/llama.cpp/llava/llava.cpp@@ -9,32 +9,295 @@#include <cstdio>#include <cstdlib>#include <vector>+#include <numeric>++// RGB uint8 image+struct clip_image_u8 {+ int nx;+ int ny;++ std::vector<uint8_t> buf;+};++// RGB float32 image (NHWC)+// Memory layout: RGBRGBRGB...+struct clip_image_f32 {+ int nx;+ int ny;++ std::vector<float> buf;+};++struct clip_image_grid_shape {+ int first;+ int second;+};++/**+ * Selects the best resolution from a list of possible resolutions based on the original size.+ *+ * @param original_size The original size of the image in the format (width, height).+ * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].+ * @return The best fit resolution in the format (width, height).+ */+static std::pair<int, int> select_best_resolution(const std::pair<int, int>& original_size, const std::vector<std::pair<int, int>>& possible_resolutions) {+ int original_width = original_size.first;+ int original_height = original_size.second;++ std::pair<int, int> best_fit;+ int max_effective_resolution = 0;+ int min_wasted_resolution = std::numeric_limits<int>::max();++ for (const auto& resolution : possible_resolutions) {+ int width = resolution.first;+ int height = resolution.second;+ float scale = std::min(static_cast<float>(width) / original_width, static_cast<float>(height) / original_height);+ int downscaled_width = static_cast<int>(original_width * scale);+ int downscaled_height = static_cast<int>(original_height * scale);+ int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height);+ int wasted_resolution = (width * height) - effective_resolution;+ // fprintf(stderr, "resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution);+ if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) {+ max_effective_resolution = effective_resolution;+ min_wasted_resolution = wasted_resolution;+ best_fit = resolution;+ }+ }++ return best_fit;+}++/**+ * @brief Get the anyres image grid shape object+ *+ * @param image_size+ * @param grid_pinpoints+ * @param image_patch_size+ * @return <int, int>+ */+static struct clip_image_grid_shape get_anyres_image_grid_shape(const std::pair<int, int> & image_size, const std::vector<std::pair<int, int>> & grid_pinpoints, int image_patch_size) {+ /**+ Conversion from gguf flat array to vector:+ std::vector<std::pair<int, int>> possible_resolutions;+ for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) {+ possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]});+ }+ */+ auto best_resolution = select_best_resolution(image_size, grid_pinpoints);+ return {best_resolution.first / image_patch_size, best_resolution.second / image_patch_size};+}++// Take the image segments in a grid configuration and return the embeddings and the number of embeddings into preallocated memory (image_embd_out)+static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector<float *> & image_embd_v, struct clip_image_grid_shape grid_shape, float * image_embd_out, int * n_img_pos_out) {+ struct {+ struct ggml_tensor * newline;+ struct ggml_context * ctx;+ } model;++ const int32_t image_size = clip_image_size(ctx_clip);+ const int32_t patch_size = clip_patch_size(ctx_clip);++ int32_t num_patches_per_side = image_size / patch_size; // 336 / 14 = 24 - used for embedding-patching boxes (24*24 = 576 patches)++ int num_patches_width = grid_shape.first; // grid 1-4+ int num_patches_height = grid_shape.second; // grid 1-4++ const size_t num_images = num_patches_width * num_patches_height + 1;++ // TODO: size calculation is not calculated - it's only tens of MB+ size_t ctx_size = 0;++ {+ ctx_size += clip_embd_nbytes(ctx_clip) * num_images * 8; // image_features+ ctx_size += 1024*1024 * ggml_type_size(GGML_TYPE_F32);+ }++ struct ggml_init_params params {+ /*.mem_size =*/ ctx_size,+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ false, // NOTE: this should be false when using the legacy API+ };++ // Python reference code for full unpad:+ /*+ base_image_feature = image_feature[0]+ image_feature = image_feature[1:]+ image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()+ image_feature = image_feature.flatten(1, 2).flatten(2, 3)+ image_feature = unpad_image(image_feature, image_sizes[image_idx])+ image_feature = torch.cat((+ image_feature,+ self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1)+ ), dim=-1)+ image_feature = image_feature.flatten(1, 2).transpose(0, 1)+ image_feature = torch.cat((base_image_feature, image_feature), dim=0)+ */+ // We now have two options: unpad or no unpad. Unpad removes tokens for faster llm eval.+ // In terms of result quality it appears to make no difference, so we'll start with the easier approach given 5D tensors are not supported in ggml yet.+ // Without unpad we have to split the sub-image embeddings into patches of 24 features each and permute them.+ // Once all images are processed to prepended the base_image_features without any changes.++ // Pytorch reference simplified, modified for ggml compatibility - confirmed identical output in python (for a 2x2 grid image (676x676 scaling))+ /*+ image_feature = image_feature.view(2, 2, 24, 24, 4096)+ image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous()+ image_feature = image_feature.view(2, 24, 2, 24, 4096)+ image_feature = image_feature.flatten(0, 3)++ // Reshape to 4D tensor by merging the last two dimensions+ image_feature = image_feature.view(2, 2, 24, 24*4096)+ image_feature = image_feature.permute(0, 2, 1, 3).contiguous()+ image_feature = image_feature.view(-1, 4096)+ */++ model.ctx = ggml_init(params);++ ggml_tensor * newline_tmp = clip_get_newline_tensor(ctx_clip);+ model.newline = ggml_new_tensor_1d(model.ctx, GGML_TYPE_F32, newline_tmp->ne[0]);+ if (newline_tmp->backend != GGML_BACKEND_CPU) {+ if (newline_tmp->buffer == NULL) {+ fprintf(stderr, "newline_tmp tensor buffer is NULL\n");+ }+ ggml_backend_tensor_get(newline_tmp, model.newline->data, 0, ggml_nbytes(newline_tmp));+ } else {+ model.newline->data = newline_tmp->data;+ if (model.newline->data == NULL) {+ fprintf(stderr, "newline_tmp tensor data is NULL\n");+ }+ }++ struct ggml_tensor * image_features = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, clip_n_mmproj_embd(ctx_clip), clip_n_patches(ctx_clip), num_images - 1); // example: 4096 x 576 x 4+ // ggml_tensor_printf(image_features,"image_features",__LINE__,false,false);+ // fill it with the image embeddings, ignoring the base+ for (size_t i = 1; i < num_images; i++) {+ size_t offset = (i-1) * clip_embd_nbytes(ctx_clip);+ memcpy((uint8_t *)(image_features->data) + offset, image_embd_v[i], clip_embd_nbytes(ctx_clip));+ }++ struct ggml_cgraph * gf = ggml_new_graph(model.ctx);+ size_t size_ele = ggml_type_size(GGML_TYPE_F32);++ struct ggml_tensor *image_features_patchview = ggml_view_4d(model.ctx, image_features,+ num_patches_per_side * clip_n_mmproj_embd(ctx_clip),+ num_patches_per_side,+ num_patches_width,+ num_patches_height,+ size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip),+ size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side,+ size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side * num_patches_width, 0);+ // ggml_tensor_printf(image_features_patchview,"image_features_patchview",__LINE__,false,false);+ struct ggml_tensor *permuted_cont = ggml_cont(model.ctx, ggml_permute(model.ctx, image_features_patchview, 0, 2, 1, 3));+ /**+ At the end of each row we have to add the row_end embeddings, which are the same as the newline embeddings+ image_feature = torch.cat((+ image_feature,+ self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1).to(image_feature.device)+ ), dim=-1)+ *+ */++ // ggml_tensor_printf(permuted_cont,"permuted_cont",__LINE__,false,false);+ struct ggml_tensor *flatten = ggml_view_2d(model.ctx, permuted_cont, clip_n_mmproj_embd(ctx_clip), num_patches_height * num_patches_width * num_patches_per_side * num_patches_per_side, size_ele * clip_n_mmproj_embd(ctx_clip), 0);+ // ggml_tensor_printf(flatten,"flatten",__LINE__,false,false);+ ggml_build_forward_expand(gf, flatten);+ ggml_graph_compute_with_ctx(model.ctx, gf, 1);+ struct ggml_tensor* result = gf->nodes[gf->n_nodes - 1];++ memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as global context+ // append without newline tokens (default behavior in llava_arch when not using unpad ):+ memcpy(image_embd_out + clip_n_patches(ctx_clip) * clip_n_mmproj_embd(ctx_clip), (float*)result->data, clip_embd_nbytes(ctx_clip) * (num_images-1)); // grid patches+ *n_img_pos_out = static_cast<int>(result->ne[1]+clip_n_patches(ctx_clip));++ // Debug: Test single segments+ // Current findings: sending base image, sending a segment embedding all works similar to python+ // However, permuted embeddings do not work yet (stride issue?)+ // memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as context+ // memcpy(image_embd_out, (float*)prepared_cont->data, clip_embd_nbytes(ctx_clip)); // main image as context+ // *n_img_pos_out=576;++ ggml_free(model.ctx);+ return true;+}-#include "llama.cpp/base64.h"static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) {- clip_image_f32 * img_res = clip_image_f32_init();- if (!clip_image_preprocess(ctx_clip, img, img_res, /*pad2square =*/ true)) {+ // std::vector<clip_image_f32*> img_res_v; // format VectN x H x W x RGB (N x 336 x 336 x 3), so interleaved RGB - different to the python implementation which is N x 3 x 336 x 336+ clip_image_f32_batch img_res_v;+ img_res_v.size = 0;+ img_res_v.data = nullptr;+ if (!clip_image_preprocess(ctx_clip, img, img_res_v)) {fprintf(stderr, "%s: unable to preprocess image\n", __func__);- clip_image_f32_free(img_res);+ delete[] img_res_v.data;return false;}- *n_img_pos = clip_n_patches(ctx_clip);-const int64_t t_img_enc_start_us = ggml_time_us();- bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd);- clip_image_f32_free(img_res);- if (!encoded) {- fprintf(stderr, "Unable to encode image\n");- return false;+ const char * mm_patch_merge_type = clip_patch_merge_type(ctx_clip);++ if (strcmp(mm_patch_merge_type, "spatial_unpad") != 0) {+ // flat / default llava-1.5 type embedding+ *n_img_pos = clip_n_patches(ctx_clip);+ bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[0], image_embd); // image_embd shape is 576 x 4096+ delete[] img_res_v.data;+ if (!encoded) {+ fprintf(stderr, "Unable to encode image\n");++ return false;+ }+ } else {+ // spatial_unpad llava-1.6 type embedding+ // TODO: CLIP needs batching support - in HF the llm projection is separate after encoding, which might be a solution to quickly get batching working+ std::vector<float *> image_embd_v;+ image_embd_v.resize(img_res_v.size);+ for (size_t i = 0; i < img_res_v.size; i++) {+ image_embd_v[i] = (float *)malloc(clip_embd_nbytes(ctx_clip)); // 576 patches * 4096 embeddings * 4 bytes = 9437184+ const bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[i], image_embd_v[i]); // image data is in 3x336x336 format and will be converted to 336x336x3 inside+ if (!encoded) {+ fprintf(stderr, "Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) img_res_v.size);+ return false;+ }+ }+ const int64_t t_img_enc_batch_us = ggml_time_us();+ tinylogf("%s: %d segments encoded in %8.2f ms\n", __func__, (int)img_res_v.size, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0);++ const int32_t * image_grid = clip_image_grid(ctx_clip);++ std::vector<std::pair<int, int>> grid_pinpoints;+ for (int i = 0; i < 32 && image_grid[i] != 0; i += 2) {+ grid_pinpoints.push_back({image_grid[i], image_grid[i+1]});+ }++ // free all img_res_v - not needed anymore+ delete[] img_res_v.data;+ img_res_v.size = 0;+ img_res_v.data = nullptr;++ const int32_t image_size = clip_image_size(ctx_clip);++ struct clip_image_grid_shape grid_shape = get_anyres_image_grid_shape({img->nx,img->ny}, grid_pinpoints, image_size);++ int n_img_pos_out;+ clip_llava_handle_patches(ctx_clip, image_embd_v, grid_shape, image_embd, &n_img_pos_out);+ *n_img_pos = n_img_pos_out;++ for (size_t i = 0; i < image_embd_v.size(); i++) {+ free(image_embd_v[i]);+ }+ image_embd_v.clear();++ // debug image/segment/normalization content:+ // clip_image_u8 * tmp = clip_image_u8_init();+ // clip_image_convert_f32_to_u8(*image_feature, *tmp);+ // clip_image_save_to_bmp(*tmp, "image_feature.bmp");}+ tinylogf("%s: image embedding created: %d tokens\n", __func__, *n_img_pos);+const int64_t t_img_enc_end_us = ggml_time_us();float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;- tinylogf("%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);+ tinylogf("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);return true;}@@ -44,17 +307,16 @@ bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx *int n_llama_embd = llama_n_embd(llama_get_model(ctx_llama));auto n_image_embd = clip_n_mmproj_embd(ctx_clip);if (n_image_embd != n_llama_embd) {- printf("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);+ fprintf(stderr, "%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);return false;}return true;}-static bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) {- float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip));+bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) {+ float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*6); // TODO: base on gridsize/llava modelif (!image_embd) {fprintf(stderr, "Unable to allocate memory for image embeddings\n");- free(image_embd);return false;}@@ -88,7 +350,7 @@ bool llava_eval_image_embed(llama_context * ctx_llama, const struct llava_image_return true;}-LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) {+struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) {clip_image_u8 * img = clip_image_u8_init();if (!clip_image_load_from_bytes(image_bytes, image_bytes_length, img)) {clip_image_u8_free(img);@@ -145,7 +407,7 @@ static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, longreturn true;}-LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) {+struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) {unsigned char* image_bytes;long image_bytes_length;auto loaded = load_file_to_bytes(image_path, &image_bytes, &image_bytes_length);@@ -154,13 +416,13 @@ LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(structreturn NULL;}- auto embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length);+ llava_image_embed *embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length);free(image_bytes);return embed;}-LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed) {+void llava_image_embed_free(struct llava_image_embed * embed) {free(embed->embed);free(embed);}diff --git a/llama.cpp/llava/llava.h b/llama.cpp/llava/llava.hindex b0948f04..69e819be 100644--- a/llama.cpp/llava/llava.h+++ b/llama.cpp/llava/llava.h@@ -3,6 +3,7 @@#include "llama.cpp/ggml.h"#include "llama.cpp/llama.h"+#include "clip.h"#ifdef LLAMA_SHARED@@ -33,6 +34,8 @@ struct llava_image_embed {/** sanity check for clip <-> llava embed size match */LLAVA_API bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip);+LLAVA_API bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out);+/** build an image embed from image file bytes */LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length);/** build an image embed from a path to an image filename */diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 4fbf4289..52c1149a 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -34,6 +34,17 @@ static std::ostringstream * g_output_ss;static std::vector<llama_token> * g_output_tokens;static bool is_interacting = false;+static bool file_exists(const std::string &path) {+ std::ifstream f(path.c_str());+ return f.good();+}++static bool file_is_empty(const std::string &path) {+ std::ifstream f;+ f.exceptions(std::ifstream::failbit | std::ifstream::badbit);+ f.open(path.c_str(), std::ios::in | std::ios::binary | std::ios::ate);+ return f.tellg() == 0;+}static void write_logfile(const llama_context * ctx, const gpt_params & params, const llama_model * model,@@ -154,7 +165,7 @@ int main(int argc, char ** argv) {console::init(params.simple_io, params.use_color);atexit([]() { console::cleanup(); });- if (!params.unsecure && !llamafile_has_gpu()) {+ if (!FLAG_unsecure && !llamafile_has_gpu()) {// Enable pledge() security on Linux and OpenBSD.// - We do this *after* opening the log file for writing.// - We do this *before* loading any weights or graphdefs.@@ -220,7 +231,8 @@ int main(int argc, char ** argv) {}LOG("%s: llama backend init\n", __func__);- llama_backend_init(params.numa);+ llama_backend_init();+ llama_numa_init(params.numa);llama_model * model;llama_context * ctx;@@ -261,12 +273,12 @@ int main(int argc, char ** argv) {if (!path_session.empty()) {LOG_TEE("%s: attempting to load saved session from '%s'\n", __func__, path_session.c_str());-- // fopen to check for existing session- FILE * fp = std::fopen(path_session.c_str(), "rbe");- if (fp != NULL) {- std::fclose(fp);-+ if (!file_exists(path_session)) {+ LOG_TEE("%s: session file does not exist, will create.\n", __func__);+ } else if (file_is_empty(path_session)) {+ LOG_TEE("%s: The session file is empty. A new session will be initialized.\n", __func__);+ } else {+ // The file exists and is not emptysession_tokens.resize(n_ctx);size_t n_token_count_out = 0;if (!llama_load_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.capacity(), &n_token_count_out)) {@@ -275,10 +287,7 @@ int main(int argc, char ** argv) {}session_tokens.resize(n_token_count_out);llama_set_rng_seed(ctx, params.seed);-- LOG_TEE("%s: loaded a session with prompt size of %d tokens\n", __func__, (int) session_tokens.size());- } else {- LOG_TEE("%s: session file does not exist, will create\n", __func__);+ LOG_TEE("%s: loaded a session with prompt size of %d tokens\n", __func__, (int)session_tokens.size());}}@@ -371,6 +380,8 @@ int main(int argc, char ** argv) {// number of tokens to keep when resetting contextif (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct || params.chatml) {params.n_keep = (int)embd_inp.size();+ } else {+ params.n_keep += add_bos; // always keep the BOS token}// prefix & suffix for instruct mode@@ -390,12 +401,12 @@ int main(int argc, char ** argv) {// in instruct mode, we inject a prefix and a suffix to each input by the userif (params.instruct) {params.interactive_first = true;- params.antiprompt.push_back("### Instruction:\n\n");+ params.antiprompt.emplace_back("### Instruction:\n\n");}// similar for chatml modeelse if (params.chatml) {params.interactive_first = true;- params.antiprompt.push_back("<|im_start|>user\n");+ params.antiprompt.emplace_back("<|im_start|>user\n");}// enable interactive mode if interactive start is specified@@ -420,8 +431,8 @@ int main(int argc, char ** argv) {}}- if (params.n_keep > 0) {- LOG_TEE("%s: static prompt based on n_keep: '", __func__);+ if (params.n_keep > add_bos) {+ LOG_TEE("%s: static prompt based on n_keep: '", __func__);for (int i = 0; i < params.n_keep; i++) {LOG_TEE("%s", llama_token_to_piece(ctx, embd_inp[i]).c_str());}@@ -567,14 +578,14 @@ int main(int argc, char ** argv) {break;}- const int n_left = n_past - params.n_keep - 1;+ const int n_left = n_past - params.n_keep;const int n_discard = n_left/2;LOG("context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d, n_discard = %d\n",n_past, n_left, n_ctx, params.n_keep, n_discard);- llama_kv_cache_seq_rm (ctx, 0, params.n_keep + 1 , params.n_keep + n_discard + 1);- llama_kv_cache_seq_shift(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard);+ llama_kv_cache_seq_rm (ctx, 0, params.n_keep , params.n_keep + n_discard);+ llama_kv_cache_seq_shift(ctx, 0, params.n_keep + n_discard, n_past, -n_discard);n_past -= n_discard;diff --git a/llama.cpp/perplexity/perplexity.cpp b/llama.cpp/perplexity/perplexity.cppindex b9041ec6..1b8b7213 100644--- a/llama.cpp/perplexity/perplexity.cpp+++ b/llama.cpp/perplexity/perplexity.cpp@@ -307,7 +307,7 @@ static void process_logits(int n_vocab, const float * logits, const int * tokens}static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params & params) {- // Download: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip?ref=salesforce-research+ // Download: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip// Run `./perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw`// Output: `perplexity: 13.5106 [114/114]`// BOS tokens will be added for each chunk before eval@@ -445,7 +445,7 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parreturn perplexity_v2(ctx, params);}- // Download: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip?ref=salesforce-research+ // Download: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip// Run `./perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw`// Output: `perplexity: 13.5106 [114/114]`// BOS tokens will be added for each chunk before eval@@ -455,14 +455,14 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parstd::ofstream logits_stream;if (!params.logits_file.empty()) {- logits_stream.open(params.logits_file.c_str());+ logits_stream.open(params.logits_file.c_str(), std::ios::binary);if (!logits_stream.is_open()) {fprintf(stderr, "%s: failed to open %s for writing\n", __func__, params.logits_file.c_str());return {};}fprintf(stderr, "%s: saving all logits to %s\n", __func__, params.logits_file.c_str());logits_stream.write("_logits_", 8);- logits_stream.write((const char *)&n_ctx, sizeof(n_ctx));+ logits_stream.write(reinterpret_cast<const char *>(&n_ctx), sizeof(n_ctx));}auto tim1 = std::chrono::high_resolution_clock::now();@@ -879,7 +879,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {size_t li = hs_cur.common_prefix;for (int s = 0; s < 4; ++s) {for (size_t j = hs_cur.common_prefix; j < hs_cur.seq_tokens[s].size() - 1; j++) {- eval_pairs.push_back(std::make_pair(hs_cur.i_batch + li++, hs_cur.seq_tokens[s][j + 1]));+ eval_pairs.emplace_back(hs_cur.i_batch + li++, hs_cur.seq_tokens[s][j + 1]);}++li;}@@ -1157,13 +1157,13 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) {const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0;size_t li = n_base1 - 1;for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) {- eval_pairs.push_back(std::make_pair(task.i_batch + li++, task.seq_tokens[0][j+1]));+ eval_pairs.emplace_back(task.i_batch + li++, task.seq_tokens[0][j+1]);}const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix;const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0;li = task.seq_tokens[0].size() - task.common_prefix + n_base2 - 1;for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) {- eval_pairs.push_back(std::make_pair(task.i_batch + li++, task.seq_tokens[1][j+1]));+ eval_pairs.emplace_back(task.i_batch + li++, task.seq_tokens[1][j+1]);}}compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results);@@ -1522,7 +1522,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & paramssize_t li = cur_task.common_prefix;for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) {for (size_t j = cur_task.common_prefix; j < cur_task.seq_tokens[s].size() - 1; j++) {- eval_pairs.push_back(std::make_pair(cur_task.i_batch + li++, cur_task.seq_tokens[s][j + 1]));+ eval_pairs.emplace_back(cur_task.i_batch + li++, cur_task.seq_tokens[s][j + 1]);}++li;}@@ -1621,7 +1621,7 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {uint32_t n_ctx;in.read((char *)&n_ctx, sizeof(n_ctx));if (n_ctx > llama_n_ctx(ctx)) {- fprintf(stderr, "%s: %s has been computed with %d, while the current context is %d. Increase it with -c and retry\n",+ fprintf(stderr, "%s: %s has been computed with %u, while the current context is %d. Increase it with -c and retry\n",__func__, params.logits_file.c_str(), n_ctx, params.n_ctx);}@@ -1823,7 +1823,8 @@ int main(int argc, char ** argv) {params.prompt = gpt_random_prompt(rng);}- llama_backend_init(params.numa);+ llama_backend_init();+ llama_numa_init(params.numa);llama_model * model;llama_context * ctx;diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex 570566d4..a380cf2e 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -28,20 +28,23 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {{ "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 4.70G, +0.0349 ppl @ LLaMA-v1-7B", },{ "IQ2_XXS",LLAMA_FTYPE_MOSTLY_IQ2_XXS," 2.06 bpw quantization", },{ "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", },+ { "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S, " 1.56 bpw quantization", },{ "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.63G, +0.6717 ppl @ LLaMA-v1-7B", },{ "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.16G, +9.0634 ppl @ LLaMA-v1-7B", },+ { "IQ3_XXS",LLAMA_FTYPE_MOSTLY_IQ3_XXS," 3.06 bpw quantization", },{ "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" },{ "Q3_K_XS",LLAMA_FTYPE_MOSTLY_Q3_K_XS,"3-bit extra small quantization" , },{ "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 2.75G, +0.5551 ppl @ LLaMA-v1-7B", },{ "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.07G, +0.2496 ppl @ LLaMA-v1-7B", },{ "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 3.35G, +0.1764 ppl @ LLaMA-v1-7B", },+ { "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL, " 4.25 bpw non-linear quantization", },{ "Q4_K", LLAMA_FTYPE_MOSTLY_Q4_K_M, "alias for Q4_K_M", },{ "Q4_K_S", LLAMA_FTYPE_MOSTLY_Q4_K_S, " 3.59G, +0.0992 ppl @ LLaMA-v1-7B", },{ "Q4_K_M", LLAMA_FTYPE_MOSTLY_Q4_K_M, " 3.80G, +0.0532 ppl @ LLaMA-v1-7B", },{ "Q5_K", LLAMA_FTYPE_MOSTLY_Q5_K_M, "alias for Q5_K_M", },{ "Q5_K_S", LLAMA_FTYPE_MOSTLY_Q5_K_S, " 4.33G, +0.0400 ppl @ LLaMA-v1-7B", },{ "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 4.45G, +0.0122 ppl @ LLaMA-v1-7B", },- { "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 5.15G, -0.0008 ppl @ LLaMA-v1-7B", },+ { "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 5.15G, +0.0008 ppl @ LLaMA-v1-7B", },{ "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0, " 6.70G, +0.0004 ppl @ LLaMA-v1-7B", },{ "F16", LLAMA_FTYPE_MOSTLY_F16, "13.00G @ 7B", },{ "F32", LLAMA_FTYPE_ALL_F32, "26.00G @ 7B", },@@ -231,13 +234,13 @@ int main(int argc, char ** argv) {}} else if (strcmp(argv[arg_idx], "--include-weights") == 0) {if (arg_idx < argc-1) {- included_weights.push_back(argv[++arg_idx]);+ included_weights.emplace_back(argv[++arg_idx]);} else {usage(argv[0]);}} else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) {if (arg_idx < argc-1) {- excluded_weights.push_back(argv[++arg_idx]);+ excluded_weights.emplace_back(argv[++arg_idx]);} else {usage(argv[0]);}@@ -260,7 +263,7 @@ int main(int argc, char ** argv) {params.imatrix = &imatrix_data;}- llama_backend_init(false);+ llama_backend_init();// parse command line argumentsconst std::string fname_inp = argv[arg_idx];@@ -310,9 +313,10 @@ int main(int argc, char ** argv) {}}- if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) && imatrix_data.empty()) {+ if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS ||+ params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) && imatrix_data.empty()) {fprintf(stderr, "\n===============================================================================================\n");- fprintf(stderr, "Please do not use IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n");+ fprintf(stderr, "Please do not use IQ1_S, IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n");fprintf(stderr, "===============================================================================================\n\n\n");return 1;}diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cppindex c3c12d71..7055ef56 100644--- a/llama.cpp/sampling.cpp+++ b/llama.cpp/sampling.cpp@@ -106,15 +106,10 @@ std::string llama_sampling_print(const llama_sampling_params & params) {std::string llama_sampling_order_print(const llama_sampling_params & params) {std::string result = "CFG -> Penalties ";if (params.mirostat == 0) {- for (auto s : params.samplers_sequence) {- switch (s) {- case 'k': result += "-> top_k "; break;- case 'f': result += "-> tfs_z "; break;- case 'y': result += "-> typical_p "; break;- case 'p': result += "-> top_p "; break;- case 'm': result += "-> min_p "; break;- case 't': result += "-> temp "; break;- default : break;+ for (auto sampler_type : params.samplers_sequence) {+ const auto sampler_type_name = sampler_type_to_name_string(sampler_type);+ if (!sampler_type_name.empty()) {+ result += "-> " + sampler_type_name + " ";}}} else {@@ -129,27 +124,25 @@ static void sampler_queue(struct llama_context * ctx_main,const llama_sampling_params & params,llama_token_data_array & cur_p,- size_t & min_keep) {- const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));-+ size_t min_keep) {const float temp = params.temp;const float dynatemp_range = params.dynatemp_range;const float dynatemp_exponent = params.dynatemp_exponent;- const int32_t top_k = params.top_k <= 0 ? n_vocab : params.top_k;+ const int32_t top_k = params.top_k;const float top_p = params.top_p;const float min_p = params.min_p;const float tfs_z = params.tfs_z;const float typical_p = params.typical_p;- const std::string & samplers_sequence = params.samplers_sequence;-- for (auto s : samplers_sequence) {- switch (s){- case 'k': llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep); break;- case 'f': llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep); break;- case 'y': llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break;- case 'p': llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break;- case 'm': llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break;- case 't':+ const std::vector<llama_sampler_type> & samplers_sequence = params.samplers_sequence;++ for (auto sampler_type : samplers_sequence) {+ switch (sampler_type) {+ case llama_sampler_type::TOP_K : llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep); break;+ case llama_sampler_type::TFS_Z : llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep); break;+ case llama_sampler_type::TYPICAL_P: llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break;+ case llama_sampler_type::TOP_P : llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break;+ case llama_sampler_type::MIN_P : llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break;+ case llama_sampler_type::TEMPERATURE:if (dynatemp_range > 0) {float dynatemp_min = std::max(0.0f, temp - dynatemp_range);float dynatemp_max = std::max(0.0f, temp + dynatemp_range);@@ -259,7 +252,7 @@ static llama_token llama_sampling_sample_impl(id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu);} else {// temperature sampling- size_t min_keep = std::max(1, params.n_probs);+ size_t min_keep = std::max(1, params.min_keep);sampler_queue(ctx_main, params, cur_p, min_keep);diff --git a/llama.cpp/sampling.h b/llama.cpp/sampling.hindex 88899c09..95d87539 100644--- a/llama.cpp/sampling.h+++ b/llama.cpp/sampling.h@@ -8,10 +8,21 @@#include <vector>#include <unordered_map>+// sampler types+enum class llama_sampler_type : char {+ TOP_K = 'k',+ TOP_P = 'p',+ MIN_P = 'm',+ TFS_Z = 'f',+ TYPICAL_P = 'y',+ TEMPERATURE = 't'+};+// sampling parameterstypedef struct llama_sampling_params {int32_t n_prev = 64; // number of previous tokens to rememberint32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.+ int32_t min_keep = 0; // 0 = disabled, otherwise samplers should return at least min_keep tokensint32_t top_k = 40; // <= 0 to use vocab sizefloat top_p = 0.95f; // 1.0 = disabledfloat min_p = 0.05f; // 0.0 = disabled@@ -28,7 +39,15 @@ typedef struct llama_sampling_params {float mirostat_tau = 5.00f; // target entropyfloat mirostat_eta = 0.10f; // learning ratebool penalize_nl = true; // consider newlines as a repeatable token- std::string samplers_sequence = "kfypmt"; // top_k, tail_free, typical_p, top_p, min_p, temp++ std::vector<llama_sampler_type> samplers_sequence = {+ llama_sampler_type::TOP_K,+ llama_sampler_type::TFS_Z,+ llama_sampler_type::TYPICAL_P,+ llama_sampler_type::TOP_P,+ llama_sampler_type::MIN_P,+ llama_sampler_type::TEMPERATURE+ };std::string grammar; // optional BNF-like grammar to constrain samplingdiff --git a/llama.cpp/server/oai.h b/llama.cpp/server/oai.hindex 691706c9..1ddca949 100644--- a/llama.cpp/server/oai.h+++ b/llama.cpp/server/oai.h@@ -18,7 +18,9 @@using json = nlohmann::json;inline static json oaicompat_completion_params_parse(- const json &body /* openai api json semantics */)+ const struct llama_model * model,+ const json &body, /* openai api json semantics */+ const std::string &chat_template){json llama_params;@@ -33,7 +35,7 @@ inline static json oaicompat_completion_params_parse(// https://platform.openai.com/docs/api-reference/chat/createllama_sampling_params default_sparams;llama_params["model"] = json_value(body, "model", std::string("unknown"));- llama_params["prompt"] = format_chatml(body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt'+ llama_params["prompt"] = format_chat(model, chat_template, body["messages"]);llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);llama_params["temperature"] = json_value(body, "temperature", 0.0);llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);@@ -209,3 +211,18 @@ inline static std::vector<json> format_partial_response_oaicompat(const task_resreturn std::vector<json>({ret});}++inline static json format_embeddings_response_oaicompat(const json &request, const json &embeddings)+{+ json res =+ json{+ {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},+ {"object", "list"},+ {"usage",+ json{{"prompt_tokens", 0},+ {"total_tokens", 0}}},+ {"data", embeddings}+ };+ return res;+}+diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 3f6a7be4..9a1b9ab8 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -1,27 +1,11 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi--#include "server.h"--#include <atomic>-#include <chrono>-#include <condition_variable>-#include <cstddef>-#include <thread>-#include <libc/calls/pledge.h>-#include <tool/args/args.h>-#include <libc/dce.h>-#include "llama.cpp/common.h"-#include "llama.cpp/ggml-cuda.h"-#include "llama.cpp/ggml-metal.h"-#include "llama.cpp/grammar-parser.h"#include "llama.cpp/llama.h"-#include "llama.cpp/llava/clip.h"+#include "llama.cpp/grammar-parser.h"+#include "llama.cpp/llava/llava.h"#include "llama.cpp/stb_image.h"-#include "llamafile/llamafile.h"-#include "llamafile/log.h"-#include "llamafile/version.h"+#include "utils.h"#include "oai.h"#include "macsandbox.h"@@ -30,9 +14,15 @@#include "httplib.h"#include "json.h"-#ifndef SERVER_VERBOSE-#define SERVER_VERBOSE 1-#endif+#include <cstddef>+#include <thread>+#include <chrono>+#include <condition_variable>+#include <atomic>+#include <signal.h>+#include <libc/calls/pledge.h>+#include <tool/args/args.h>+#include <libc/dce.h>using json = nlohmann::json;@@ -41,11 +31,12 @@ struct server_paramsstd::string hostname = "127.0.0.1";std::vector<std::string> api_keys;std::string public_path = "/zip/llama.cpp/server/public";+ std::string chat_template = "";int32_t port = 8080;int32_t read_timeout = 600;int32_t write_timeout = 600;bool nobrowser = false;- bool unsecure = false;+ bool slots_endpoint = true;};bool server_verbose = false;@@ -164,6 +155,7 @@ struct llama_client_slotint32_t n_decoded = 0;int32_t n_remaining = -1;int32_t i_batch = -1;+ int32_t n_predict = -1;int32_t num_prompt_tokens = 0;int32_t num_prompt_tokens_processed = 0;@@ -192,7 +184,7 @@ struct llama_client_slotllama_sampling_context *ctx_sampling = nullptr;int32_t ga_i = 0; // group-attention state- int32_t ga_n = 1;// group-attention factor+ int32_t ga_n = 1; // group-attention factorint32_t ga_w = 512; // group-attention widthint32_t n_past_se = 0; // self-extend@@ -226,7 +218,8 @@ struct llama_client_slotsent_token_probs_index = 0;infill = false;ga_i = 0;- n_past_se = 0;+ n_past_se = 0;+generated_token_probs.clear();for (slot_image & img : images)@@ -340,6 +333,7 @@ struct llama_server_context// slots / clientsstd::vector<llama_client_slot> slots;+ json default_generation_settings_for_props;llama_server_queue queue_tasks;llama_server_response queue_results;@@ -413,6 +407,7 @@ struct llama_server_contextslot.id = i;slot.n_ctx = n_ctx_slot;+ slot.n_predict = params.n_predict;LOG_TEE(" -> Slot %i - max context: %i\n", slot.id, n_ctx_slot);@@ -436,11 +431,10 @@ struct llama_server_contextslots.push_back(slot);}- batch = llama_batch_init(n_ctx, 0, params.n_parallel);+ default_generation_settings_for_props = get_formated_generation(slots.front());+ default_generation_settings_for_props["seed"] = -1;- // empty system prompt- system_prompt = "";- system_tokens.clear();+ batch = llama_batch_init(n_ctx, 0, params.n_parallel);}std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const@@ -526,27 +520,39 @@ struct llama_server_contextslot->oaicompat_model = "";}- slot->params.stream = json_value(data, "stream", false);- slot->params.cache_prompt = json_value(data, "cache_prompt", false);- slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict);- slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);- slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);- slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);- slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);- slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);- slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);- slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);- slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);- slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);- slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);- slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);- slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);- slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);- slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);- slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep);- slot->params.seed = json_value(data, "seed", default_params.seed);- slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar);- slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);+ slot->params.stream = json_value(data, "stream", false);+ slot->params.cache_prompt = json_value(data, "cache_prompt", false);+ slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict);+ slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);+ slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);+ slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);+ slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);+ slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);+ slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);+ slot->sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);+ slot->sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);+ slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);+ slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);+ slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);+ slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);+ slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);+ slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);+ slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);+ slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);+ slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep);+ slot->params.seed = json_value(data, "seed", default_params.seed);+ slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar);+ slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);+ slot->sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);++ if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) {+ // Might be better to reject the request with a 400 ?+ LOG_WARNING("Max tokens to predict exceeds server configuration", {+ {"params.n_predict", slot->params.n_predict},+ {"slot.n_predict", slot->n_predict},+ });+ slot->params.n_predict = slot->n_predict;+ }// infillif (data.count("input_prefix") != 0)@@ -625,18 +631,36 @@ struct llama_server_contextconst int n_vocab = llama_n_vocab(model);for (const auto &el : *logit_bias){- if (el.is_array() && el.size() == 2 && el[0].is_number_integer())+ if (el.is_array() && el.size() == 2){- llama_token tok = el[0].get<llama_token>();- if (tok >= 0 && tok < n_vocab)+ float bias;+ if (el[1].is_number())+ {+ bias = el[1].get<float>();+ }+ else if (el[1].is_boolean() && !el[1].get<bool>()){- if (el[1].is_number())+ bias = -INFINITY;+ }+ else+ {+ continue;+ }++ if (el[0].is_number_integer())+ {+ llama_token tok = el[0].get<llama_token>();+ if (tok >= 0 && tok < n_vocab){- slot->sparams.logit_bias[tok] = el[1].get<float>();+ slot->sparams.logit_bias[tok] = bias;}- else if (el[1].is_boolean() && !el[1].get<bool>())+ }+ else if (el[0].is_string())+ {+ auto toks = llama_tokenize(model, el[0].get<std::string>(), false);+ for (auto tok : toks){- slot->sparams.logit_bias[tok] = -INFINITY;+ slot->sparams.logit_bias[tok] = bias;}}}@@ -657,6 +681,24 @@ struct llama_server_context}}+ const auto &samplers_sequence = data.find("samplers");+ if (samplers_sequence != data.end() && samplers_sequence->is_array())+ {+ std::vector<std::string> sampler_names;+ for (const auto &sampler_name : *samplers_sequence)+ {+ if (sampler_name.is_string())+ {+ sampler_names.emplace_back(sampler_name);+ }+ }+ slot->sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);+ }+ else+ {+ slot->sparams.samplers_sequence = default_sparams.samplers_sequence;+ }+if (multimodal){const auto &images_data = data.find("image_data");@@ -746,27 +788,30 @@ struct llama_server_context}void update_system_prompt() {- system_tokens = ::llama_tokenize(ctx, system_prompt, add_bos_token);+ kv_cache_clear();+ system_tokens.clear();- llama_batch_clear(batch);+ if (!system_prompt.empty()) {+ system_tokens = ::llama_tokenize(ctx, system_prompt, add_bos_token);- kv_cache_clear();+ llama_batch_clear(batch);- for (int i = 0; i < (int) system_tokens.size(); ++i)- {- llama_batch_add(batch, system_tokens[i], i, { 0 }, false);- }+ for (int i = 0; i < (int)system_tokens.size(); ++i)+ {+ llama_batch_add(batch, system_tokens[i], i, { 0 }, false);+ }- if (llama_decode(ctx, batch) != 0)- {- LOG_TEE("%s: llama_decode() failed\n", __func__);- return;- }+ if (llama_decode(ctx, batch) != 0)+ {+ LOG_TEE("%s: llama_decode() failed\n", __func__);+ return;+ }- // assign the system KV cache to all parallel sequences- for (int32_t i = 1; i < params.n_parallel; ++i)- {- llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size());+ // assign the system KV cache to all parallel sequences+ for (int32_t i = 1; i < params.n_parallel; ++i)+ {+ llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size());+ }}LOG_TEE("system prompt updated\n");@@ -788,10 +833,8 @@ struct llama_server_contextname_user = sys_props.value("anti_prompt", "");name_assistant = sys_props.value("assistant_name", "");- if (slots.size() > 0)- {- notify_system_prompt_changed();- }++ notify_system_prompt_changed();}static size_t find_stopping_strings(const std::string &text, const size_t last_token_size,@@ -949,28 +992,13 @@ struct llama_server_context{continue;}- clip_image_f32 * img_res = clip_image_f32_init();- if (!clip_image_preprocess(clp_ctx, img.img_data, img_res, /*pad2square =*/ true))- {++ if (!llava_image_embed_make_with_clip_img(clp_ctx, params.n_threads, img.img_data, &img.image_embedding, &img.image_tokens)) {LOG_TEE("Error processing the given image");- clip_free(clp_ctx);- return false;- }- img.image_tokens = clip_n_patches(clp_ctx);- img.image_embedding = (float *)malloc(clip_embd_nbytes(clp_ctx));- if (!img.image_embedding)- {- LOG_TEE("Unable to allocate memory for image embeddings\n");- clip_free(clp_ctx);return false;}- LOG_TEE("slot %i - encoding image [id: %i]\n", slot.id, img.id);- if (!clip_image_encode(clp_ctx, params.n_threads, img_res, img.image_embedding))- {- LOG_TEE("Unable to encode image\n");- return false;- }- clip_image_f32_free(img_res);++img.request_encode_image = false;}@@ -989,21 +1017,25 @@ struct llama_server_contextqueue_results.send(res);}- json get_model_props()- {- return get_formated_generation(slots[0]);- }-json get_formated_generation(llama_client_slot &slot){const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() &&eos_bias->second < 0.0f && std::isinf(eos_bias->second);+ std::vector<std::string> samplers_sequence;+ for (const auto &sampler_type : slot.sparams.samplers_sequence)+ {+ samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type));+ }+return json {{"n_ctx", slot.n_ctx},+ {"n_predict", slot.n_predict},{"model", params.model_alias},{"seed", slot.params.seed},{"temperature", slot.sparams.temp},+ {"dynatemp_range", slot.sparams.dynatemp_range},+ {"dynatemp_exponent", slot.sparams.dynatemp_exponent},{"top_k", slot.sparams.top_k},{"top_p", slot.sparams.top_p},{"min_p", slot.sparams.min_p},@@ -1026,7 +1058,9 @@ struct llama_server_context{"stream", slot.params.stream},{"logit_bias", slot.sparams.logit_bias},{"n_probs", slot.sparams.n_probs},+ {"min_keep", slot.sparams.min_keep},{"grammar", slot.sparams.grammar},+ {"samplers", samplers_sequence}};}@@ -1165,13 +1199,30 @@ struct llama_server_contexttask.multitask_id = multitask_id;// when a completion task's prompt array is not a singleton, we split it into multiple requests- if (task.data.count("prompt") && task.data.at("prompt").size() > 1)- {- split_multiprompt_task(task_id, task);- }-// otherwise, it's a single-prompt task, we actually queue it- queue_tasks.post(task);+ // if there's numbers in the prompt array it will be treated as an array of tokens+ if (task.data.count("prompt") != 0 && task.data.at("prompt").size() > 1) {+ bool numbers = false;+ for (const auto& e : task.data.at("prompt")) {+ if (e.is_number()) {+ numbers = true;+ break;+ }+ }++ // NOTE: split_multiprompt_task() does not handle a mix of strings and numbers,+ // it will completely stall the server. I don't know where the bug for this is.+ //+ // if there are numbers, it needs to be treated like a single prompt,+ // queue_tasks handles a mix of strings and numbers just fine.+ if (numbers) {+ queue_tasks.post(task);+ } else {+ split_multiprompt_task(task_id, task);+ }+ } else {+ queue_tasks.post(task);+ }}// for multiple images processing@@ -1234,7 +1285,7 @@ struct llama_server_contextstd::vector<llama_token> append_tokens = tokenize(json_prompt, false); // has next imagefor (int i = 0; i < (int) append_tokens.size(); ++i){- llama_batch_add(batch, append_tokens[i], slot.n_past, { slot.id }, true);+ llama_batch_add(batch, append_tokens[i], system_tokens.size() + slot.n_past, { slot.id }, true);slot.n_past += 1;}}@@ -1253,7 +1304,10 @@ struct llama_server_contextvoid split_multiprompt_task(int multitask_id, task_server& multiprompt_task){int prompt_count = multiprompt_task.data.at("prompt").size();- assert(prompt_count > 1);+ if (prompt_count <= 1) {+ send_error(multiprompt_task, "error while handling multiple prompts");+ return;+ }// generate all the ID for subtaskstd::vector<int> subtask_ids(prompt_count);@@ -1302,6 +1356,8 @@ struct llama_server_contextfor (llama_client_slot &slot : slots){slot.cache_tokens.clear();+ slot.n_past = 0;+ slot.n_past_se = 0;}}@@ -1332,6 +1388,46 @@ struct llama_server_contextcase TASK_TYPE_NEXT_RESPONSE: {// do nothing} break;+ case TASK_TYPE_SLOTS_DATA: {+ json slots_data = json::array();+ int n_idle_slots = 0;+ int n_processing_slots = 0;++ for (llama_client_slot &slot: slots) {+ if (slot.available()) {+ n_idle_slots++;+ } else {+ n_processing_slots++;+ }+ json slot_data = get_formated_generation(slot);+ slot_data["id"] = slot.id;+ slot_data["task_id"] = slot.task_id;+ slot_data["state"] = slot.state;+ slot_data["prompt"] = slot.prompt;+ slot_data["next_token"] = {+ {"has_next_token", slot.has_next_token},+ {"n_remain", slot.n_remaining},+ {"num_tokens_predicted", slot.n_decoded},+ {"stopped_eos", slot.stopped_eos},+ {"stopped_word", slot.stopped_word},+ {"stopped_limit", slot.stopped_limit},+ {"stopping_word", slot.stopping_word},+ };+ slots_data.push_back(slot_data);+ }+ LOG_TEE("task %i - slots data: idle=%i processing=%i\n", task.id, n_idle_slots, n_processing_slots);+ task_result res;+ res.id = task.id;+ res.multitask_id = task.multitask_id;+ res.stop = true;+ res.error = false;+ res.result_json = {+ { "idle", n_idle_slots },+ { "processing", n_processing_slots },+ { "slots", slots_data }+ };+ queue_results.send(res);+ } break;}}@@ -1371,28 +1467,29 @@ struct llama_server_contextkv_cache_clear();}return true;- } else {- task_server task;- task.type = TASK_TYPE_NEXT_RESPONSE;- task.target_id = -1;- queue_tasks.post(task);}+ task_server task;+ task.type = TASK_TYPE_NEXT_RESPONSE;+ task.target_id = -1;+ queue_tasks.post(task);+for (llama_client_slot &slot : slots){if (slot.ga_n == 1){- if (slot.is_processing() && slot.cache_tokens.size() >= (size_t) slot.n_ctx)+ if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx){// Shift context- const int n_left = slot.n_past - slot.params.n_keep - 1;+ const int n_keep = slot.params.n_keep + add_bos_token;+ const int n_left = system_tokens.size() + slot.n_past - n_keep;const int n_discard = n_left / 2;- LOG_TEE("slot %d: context shift - n_keep = %d, n_left = %d, n_discard = %d\n", slot.id, slot.params.n_keep, n_left, n_discard);- llama_kv_cache_seq_rm (ctx, slot.id, slot.params.n_keep + 1 , slot.params.n_keep + n_discard + 1);- llama_kv_cache_seq_shift(ctx, slot.id, slot.params.n_keep + 1 + n_discard, slot.n_past, -n_discard);+ LOG_TEE("slot %d: context shift - n_keep = %d, n_left = %d, n_discard = %d\n", slot.id, n_keep, n_left, n_discard);+ llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard);+ llama_kv_cache_seq_shift(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);- for (size_t i = slot.params.n_keep + 1 + n_discard; i < slot.cache_tokens.size(); i++)+ for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++){slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];}@@ -1405,7 +1502,7 @@ struct llama_server_contextLOG_VERBOSE("context shift", {{ "n_ctx", n_ctx },- { "n_keep", params.n_keep },+ { "n_keep", n_keep },{ "n_left", n_left },});}@@ -1436,8 +1533,10 @@ struct llama_server_contextslot.i_batch = batch.n_tokens;const int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past;- llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true);+ // TODO: we always have to take into account the "system_tokens"+ // this is not great and needs to be improved somehow+ llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true);slot.n_past += 1;}@@ -1488,8 +1587,8 @@ struct llama_server_contextprefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS- prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));- prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());+ prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));+ prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());prefix_tokens.push_back(llama_token_middle(model));prompt_tokens = prefix_tokens;}@@ -1571,10 +1670,6 @@ struct llama_server_contextLOG_TEE("slot %d : in cache: %i tokens | to process: %i tokens\n", slot.id, slot.n_past, slot.num_prompt_tokens_processed);}- LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past);-- llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1);-slot.cache_tokens = prompt_tokens;if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0)@@ -1588,9 +1683,13 @@ struct llama_server_context}}+ LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past);++ llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1);+LOG_VERBOSE("prompt ingested", {- {"n_past", slot.n_past},- {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},+ {"n_past", slot.n_past},+ {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},{"to_eval", tokens_to_str(ctx, slot.cache_tokens.cbegin() + slot.n_past, slot.cache_tokens.cend())},});@@ -1598,10 +1697,13 @@ struct llama_server_context// process the prefix of first imagestd::vector<llama_token> prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens;+int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past;- int ga_i = slot.ga_i;++ int32_t ga_i = slot.ga_i;int32_t ga_n = slot.ga_n;int32_t ga_w = slot.ga_w;+for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past){if (slot.ga_n != 1)@@ -1613,7 +1715,7 @@ struct llama_server_context}}llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, {slot.id }, false);- slot_npast += 1;+ slot_npast++;}if (has_images && !ingest_images(slot, n_batch))@@ -1673,6 +1775,7 @@ struct llama_server_contextslot.n_past_se += n_tokens;}}+llama_batch batch_view ={n_tokens,@@ -1787,53 +1890,61 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n");printf(" not recommended: doubles context memory required and no measurable increase in quality\n");- if (llama_mlock_supported())+ if (llama_supports_mlock()){- printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");+ printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");}- if (llama_mmap_supported())+ if (llama_supports_mmap()){- printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");+ printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");+ }+ printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n");+ printf(" - distribute: spread execution evenly over all nodes\n");+ printf(" - isolate: only spawn threads on CPUs on the node that execution started on\n");+ printf(" - numactl: use the CPU map provided my numactl\n");+ if (llama_supports_gpu_offload()) {+ printf(" -ngl N, --n-gpu-layers N\n");+ printf(" number of layers to store in VRAM\n");+ printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");+ printf(" how to split the model across multiple GPUs, one of:\n");+ printf(" - none: use one GPU only\n");+ printf(" - layer (default): split layers and KV across GPUs\n");+ printf(" - row: split rows across GPUs\n");+ printf(" -ts SPLIT --tensor-split SPLIT\n");+ printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");+ printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");+ printf(" or for intermediate results and KV (with split-mode = row)\n");}- printf(" --numa attempt optimizations that help on some NUMA systems\n");-#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD- printf(" -ngl N, --n-gpu-layers N\n");- printf(" number of layers to store in VRAM\n");- printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");- printf(" how to split the model across multiple GPUs, one of:\n");- printf(" - none: use one GPU only\n");- printf(" - layer (default): split layers and KV across GPUs\n");- printf(" - row: split rows across GPUs\n");- printf(" -ts SPLIT --tensor-split SPLIT\n");- printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");- printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");- printf(" or for intermediate results and KV (with split-mode = row)\n");-#endifprintf(" -m FNAME, --model FNAME\n");- printf(" model path (default: %s)\n", params.model.c_str());+ printf(" model path (default: %s)\n", params.model.c_str());printf(" -a ALIAS, --alias ALIAS\n");- printf(" set an alias for the model, will be added as `model` field in completion response\n");- printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");- printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");- printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());- printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);- printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());- printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");- printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");- printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);- printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");- printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);- printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");- printf(" -spf FNAME, --system-prompt-file FNAME\n");- printf(" Set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n");- printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");- printf(" --log-disable disables logging to a file.\n");+ printf(" set an alias for the model, will be added as `model` field in completion response\n");+ printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");+ printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");+ printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());+ printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);+ printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());+ printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");+ printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");+ printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);+ printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");+ printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);+ printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");+ printf(" -spf FNAME, --system-prompt-file FNAME\n");+ printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n");+ printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");+ printf(" --log-disable disables logging to a file.\n");+ printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n");printf("\n");+ printf(" -n, --n-predict maximum tokens to predict (default: %d)\n", params.n_predict);printf(" --override-kv KEY=TYPE:VALUE\n");- printf(" advanced option to override model metadata by key. may be specified multiple times.\n");- printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");- printf(" -gan N, --grp-attn-n N Set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`");- printf(" -gaw N, --grp-attn-w N Set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`");+ printf(" advanced option to override model metadata by key. may be specified multiple times.\n");+ printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");+ printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`");+ printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`");+ printf(" --chat-template JINJA_TEMPLATE\n");+ printf(" set custom jinja chat template (default: template taken from model's metadata)\n");+ printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n");printf("\n");}@@ -1882,7 +1993,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}- sparams.api_keys.push_back(argv[i]);+ sparams.api_keys.emplace_back(argv[i]);}else if (arg == "--api-key-file"){@@ -2064,13 +2175,13 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}-#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD- params.n_gpu_layers = std::stoi(argv[i]);-#else- LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "+ if (llama_supports_gpu_offload()) {+ params.n_gpu_layers = std::stoi(argv[i]);+ } else {+ LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. ""See main README.md for information on enabling GPU BLAS support",{{"n_gpu_layers", params.n_gpu_layers}});-#endif+ }}else if (arg == "--split-mode" || arg == "-sm"){@@ -2109,9 +2220,9 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,const std::regex regex{R"([,/]+)"};std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1};std::vector<std::string> split_arg{it, {}};- GGML_ASSERT(split_arg.size() <= LLAMA_MAX_DEVICES);+ GGML_ASSERT(split_arg.size() <= llama_max_devices());- for (size_t i_device = 0; i_device < LLAMA_MAX_DEVICES; ++i_device)+ for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device){if (i_device < split_arg.size()){@@ -2143,7 +2254,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}- params.lora_adapter.push_back(std::make_tuple(argv[i], 1.0f));+ params.lora_adapter.emplace_back(argv[i], 1.0f);params.use_mmap = false;}else if (arg == "--lora-scaled")@@ -2159,7 +2270,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}- params.lora_adapter.push_back(std::make_tuple(lora_adapter, std::stof(argv[i])));+ params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));params.use_mmap = false;}else if (arg == "--lora-base")@@ -2185,7 +2296,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}else if (arg == "--unsecure"){- sparams.unsecure = true;+ FLAG_unsecure = true;}else if (arg == "--server"){@@ -2220,9 +2331,17 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,{params.use_mmap = false;}- else if (arg == "--numa")- {- params.numa = true;+ else if (arg == "--numa") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ } else {+ std::string value(argv[i]);+ /**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }+ else { invalid_param = true; break; }+ }}else if (arg == "--embedding"){@@ -2283,6 +2402,25 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,log_set_target(stdout);LOG_INFO("logging to file is disabled.", {});}+ else if (arg == "--slots-endpoint-disable")+ {+ sparams.slots_endpoint = false;+ }+ else if (arg == "--chat-template")+ {+ if (++i >= argc)+ {+ invalid_param = true;+ break;+ }+ if (!verify_custom_template(argv[i])) {+ fprintf(stderr, "error: the supplied chat template is not supported: %s\n", argv[i]);+ fprintf(stderr, "note: llama.cpp does not use jinja parser, we only support commonly used templates\n");+ invalid_param = true;+ break;+ }+ sparams.chat_template = argv[i];+ }else if (arg == "--override-kv"){if (++i >= argc) {@@ -2335,11 +2473,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}params.embedding = true; // [jart] #243 always enable embedding mode-params.n_gpu_layers = llamafile_gpu_layers(params.n_gpu_layers);if (!params.kv_overrides.empty()) {- params.kv_overrides.emplace_back(llama_model_kv_override());+ params.kv_overrides.emplace_back();params.kv_overrides.back().key[0] = 0;}@@ -2424,6 +2561,9 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con}}+std::function<void(int)> shutdown_handler;+inline void signal_handler(int signal) { shutdown_handler(signal); }+static const char *sockaddr2str(const struct sockaddr *sa, char *buf, size_t size) {if (sa->sa_family == AF_INET) {return inet_ntop(AF_INET, &(((const struct sockaddr_in *)sa)->sin_addr),@@ -2455,7 +2595,8 @@ int server_cli(int argc, char **argv)params.model_alias = params.model;}- llama_backend_init(params.numa);+ llama_backend_init();+ llama_numa_init(params.numa);LOG_INFO("build info", {{"build", LLAMA_BUILD_NUMBER},{"commit", LLAMA_COMMIT}});@@ -2481,13 +2622,44 @@ int server_cli(int argc, char **argv)res.set_header("Access-Control-Allow-Headers", "*");});- svr.Get("/health", [&](const httplib::Request&, httplib::Response& res) {+ svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) {server_state current_state = state.load();switch(current_state) {- case SERVER_STATE_READY:- res.set_content(R"({"status": "ok"})", "application/json");+ case SERVER_STATE_READY: {+ // request slots data using task queue+ task_server task;+ task.id = llama.queue_tasks.get_new_id();+ task.type = TASK_TYPE_SLOTS_DATA;+ task.target_id = -1;++ llama.queue_results.add_waiting_task_id(task.id);+ llama.queue_tasks.post(task);++ // get the result+ task_result result = llama.queue_results.recv(task.id);+ llama.queue_results.remove_waiting_task_id(task.id);++ int n_idle_slots = result.result_json["idle"];+ int n_processing_slots = result.result_json["processing"];++ json health = {+ {"status", "ok"},+ {"slots_idle", n_idle_slots},+ {"slots_processing", n_processing_slots}};res.status = 200; // HTTP OK+ if (sparams.slots_endpoint && req.has_param("include_slots")) {+ health["slots"] = result.result_json["slots"];+ }++ if (n_idle_slots == 0) {+ health["status"] = "no slot available";+ if (req.has_param("fail_on_no_slot")) {+ res.status = 503; // HTTP Service Unavailable+ }+ }+ res.set_content(health.dump(), "application/json");break;+ }case SERVER_STATE_LOADING_MODEL:res.set_content(R"({"status": "loading model"})", "application/json");res.status = 503; // HTTP Service Unavailable@@ -2499,6 +2671,26 @@ int server_cli(int argc, char **argv)}});+ if (sparams.slots_endpoint) {+ svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) {+ // request slots data using task queue+ task_server task;+ task.id = llama.queue_tasks.get_new_id();+ task.type = TASK_TYPE_SLOTS_DATA;+ task.target_id = -1;++ llama.queue_results.add_waiting_task_id(task.id);+ llama.queue_tasks.post(task);++ // get the result+ task_result result = llama.queue_results.recv(task.id);+ llama.queue_results.remove_waiting_task_id(task.id);++ res.set_content(result.result_json["slots"].dump(), "application/json");+ res.status = 200; // HTTP OK+ });+ }+svr.set_logger(log_server_request);svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep)@@ -2625,7 +2817,7 @@ int server_cli(int argc, char **argv)llamafile_launch_browser(url);}- if (!sparams.unsecure) {+ if (!FLAG_unsecure) {if (IsXnu()) {// Cosmopolitan libc explicitly does not support cosmo_dlopen on x64// macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt@@ -2713,7 +2905,9 @@ int server_cli(int argc, char **argv)res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));json data = {{ "user_name", llama.name_user.c_str() },- { "assistant_name", llama.name_assistant.c_str() }+ { "assistant_name", llama.name_assistant.c_str() },+ { "default_generation_settings", llama.default_generation_settings_for_props },+ { "total_slots", llama.params.n_parallel }};res.set_content(data.dump(), "application/json; charset=utf-8");});@@ -2817,13 +3011,13 @@ int server_cli(int argc, char **argv)// TODO: add mount point without "/v1" prefix -- how?- svr.Post("/v1/chat/completions", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)+ svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));if (!validate_api_key(req, res)) {return;}- json data = oaicompat_completion_params_parse(json::parse(req.body));+ json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template);const int task_id = llama.queue_tasks.get_new_id();llama.queue_results.add_waiting_task_id(task_id);@@ -2964,12 +3158,6 @@ int server_cli(int argc, char **argv)}});- svr.Get("/model.json", [&llama](const httplib::Request &, httplib::Response &res)- {- const json data = llama.get_model_props();- return res.set_content(data.dump(), "application/json; charset=utf-8");- });-svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res){ return res.set_content("", "application/json; charset=utf-8"); });@@ -3037,6 +3225,66 @@ int server_cli(int argc, char **argv)return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");});+ svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res)+ {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ const json body = json::parse(req.body);++ json prompt;+ if (body.count("input") != 0)+ {+ prompt = body["input"];+ // batch+ if(prompt.is_array()) {+ json data = json::array();+ int i = 0;+ for (const json &elem : prompt) {+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, { {"prompt", elem}, { "n_predict", 0} }, false, true, -1);++ // get the result+ task_result result = llama.queue_results.recv(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);++ json embedding = json{+ {"embedding", json_value(result.result_json, "embedding", json::array())},+ {"index", i++},+ {"object", "embedding"}+ };+ data.push_back(embedding);+ }+ json result = format_embeddings_response_oaicompat(body, data);+ return res.set_content(result.dump(), "application/json; charset=utf-8");+ }+ }+ else+ {+ prompt = "";+ }++ // create and queue the task+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}}, false, true, -1);++ // get the result+ task_result result = llama.queue_results.recv(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);++ json data = json::array({json{+ {"embedding", json_value(result.result_json, "embedding", json::array())},+ {"index", 0},+ {"object", "embedding"}+ }}+ );++ json root = format_embeddings_response_oaicompat(body, data);++ // send the result+ return res.set_content(root.dump(), "application/json; charset=utf-8");+ });+// GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!?// "Bus error: 10" - this is on macOS, it does not crash on Linux//std::thread t2([&]()@@ -3062,8 +3310,25 @@ int server_cli(int argc, char **argv)std::placeholders::_2,std::placeholders::_3));- llama.queue_tasks.start_loop();+ shutdown_handler = [&](int) {+ llama.queue_tasks.terminate();+ };++#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))+ struct sigaction sigint_action;+ sigint_action.sa_handler = signal_handler;+ sigemptyset (&sigint_action.sa_mask);+ sigint_action.sa_flags = 0;+ sigaction(SIGINT, &sigint_action, NULL);+#elif defined (_WIN32)+ auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {+ return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;+ };+ SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);+#endif+ llama.queue_tasks.start_loop();+ svr.stop();t.join();llama_backend_free();diff --git a/llama.cpp/server/utils.h b/llama.cpp/server/utils.hindex de8188ec..4922ca14 100644--- a/llama.cpp/server/utils.h+++ b/llama.cpp/server/utils.h@@ -52,7 +52,8 @@ enum server_state {enum task_type {TASK_TYPE_COMPLETION,TASK_TYPE_CANCEL,- TASK_TYPE_NEXT_RESPONSE+ TASK_TYPE_NEXT_RESPONSE,+ TASK_TYPE_SLOTS_DATA};struct task_server {@@ -170,20 +171,47 @@ static T json_value(const json &body, const std::string &key, const T &default_v: default_value;}-inline std::string format_chatml(std::vector<json> messages)+// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid+inline bool verify_custom_template(const std::string & tmpl) {+ llama_chat_message chat[] = {{"user", "test"}};+ std::vector<char> buf(1);+ int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, buf.data(), buf.size());+ return res >= 0;+}++// Format given chat. If tmpl is empty, we take the template from model metadata+inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages){- std::ostringstream chatml_msgs;+ size_t alloc_size = 0;+ // vector holding all allocated string to be passed to llama_chat_apply_template+ std::vector<std::string> str(messages.size() * 2);+ std::vector<llama_chat_message> chat(messages.size());++ for (size_t i = 0; i < messages.size(); ++i) {+ auto &curr_msg = messages[i];+ str[i*2 + 0] = json_value(curr_msg, "role", std::string(""));+ str[i*2 + 1] = json_value(curr_msg, "content", std::string(""));+ alloc_size += str[i*2 + 1].length();+ chat[i].role = str[i*2 + 0].c_str();+ chat[i].content = str[i*2 + 1].c_str();+ }++ const char * ptr_tmpl = tmpl.empty() ? nullptr : tmpl.c_str();+ std::vector<char> buf(alloc_size * 2);++ // run the first time to get the total output length+ int32_t res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size());- for (auto it = messages.begin(); it != messages.end(); ++it) {- chatml_msgs << "<|im_start|>"- << json_value(*it, "role", std::string("user")) << '\n';- chatml_msgs << json_value(*it, "content", std::string(""))- << "<|im_end|>\n";+ // if it turns out that our buffer is too small, we resize it+ if ((size_t) res > buf.size()) {+ buf.resize(res);+ res = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size());}- chatml_msgs << "<|im_start|>assistant" << '\n';+ std::string formatted_chat(buf.data(), res);+ LOG_VERBOSE("formatted_chat", {{"text", formatted_chat.c_str()}});- return chatml_msgs.str();+ return formatted_chat;}//@@ -193,6 +221,7 @@ inline std::string format_chatml(std::vector<json> messages)struct llama_server_queue {int id = 0;std::mutex mutex_tasks;+ bool running;// queuesstd::vector<task_server> queue_tasks;std::vector<task_server> queue_tasks_deferred;@@ -215,7 +244,7 @@ struct llama_server_queue {}// Add a new task, but defer until one slot is available- void (defer)(task_server task) {+ void defer(task_server task) {std::unique_lock<std::mutex> lock(mutex_tasks);queue_tasks_deferred.push_back(std::move(task));}@@ -251,9 +280,18 @@ struct llama_server_queue {queue_tasks_deferred.clear();}- // Start the main loop. This call is blocking- [[noreturn]]+ // end the start_loop routine+ void terminate() {+ {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ running = false;+ }+ condition_tasks.notify_all();+ }++ // Start the main loop.void start_loop() {+ running = true;while (true) {// new task arrivedLOG_VERBOSE("have new task", {});@@ -297,8 +335,12 @@ struct llama_server_queue {{std::unique_lock<std::mutex> lock(mutex_tasks);if (queue_tasks.empty()) {+ if (!running) {+ LOG_VERBOSE("ending start_loop", {});+ return;+ }condition_tasks.wait(lock, [&]{- return !queue_tasks.empty();+ return (!queue_tasks.empty() || !running);});}}diff --git a/llama.cpp/stb_image.c b/llama.cpp/stb_image.cnew file mode 100644index 00000000..8ddfd1f5--- /dev/null+++ b/llama.cpp/stb_image.c@@ -0,0 +1,2 @@+#define STB_IMAGE_IMPLEMENTATION+#include "stb_image.h"diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 26283df6..8aed849d 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -15,26 +15,26 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <time.h>+#include "llama.cpp/ggml-backend-impl.h"+#include "llama.cpp/ggml-cuda.h"+#include "llama.cpp/ggml-metal.h"+#include "llamafile/log.h"+#include "llamafile/x.h"+#include <assert.h>#include <cosmo.h>#include <dlfcn.h>#include <errno.h>-#include <spawn.h>-#include <assert.h>-#include <unistd.h>-#include <stdlib.h>-#include <limits.h>-#include <signal.h>#include <libgen.h>+#include <limits.h>#include <pthread.h>-#include <sys/wait.h>-#include <sys/stat.h>+#include <signal.h>+#include <spawn.h>#include <stdatomic.h>-#include "llamafile/x.h"-#include "llamafile/log.h"-#include "llama.cpp/ggml-cuda.h"-#include "llama.cpp/ggml-metal.h"-#include "llama.cpp/ggml-backend-impl.h"+#include <stdlib.h>+#include <sys/stat.h>+#include <sys/wait.h>+#include <time.h>+#include <unistd.h>__static_yoink("llama.cpp/ggml.h");__static_yoink("llamafile/compcap.cu");@@ -55,26 +55,19 @@ __static_yoink("llama.cpp/ggml-backend-impl.h");#define NVCC_LIBS BLAS_ONLY("-lcublas"), "-lcuda"-#define NVCC_FLAGS "--shared", \- "--forward-unknown-to-host-compiler", \- "-use_fast_math", \- "--compiler-options", \- (!IsWindows() \- ? (!IsAarch64() \- ? "-fPIC -O3 -march=native -mtune=native" \- : "-fPIC -O3 -march=native -mtune=native -ffixed-x28") \- : "/nologo /EHsc /O2 /GR /MT"), \- "-DNDEBUG", \- "-DGGML_BUILD=1", \- "-DGGML_SHARED=1", \- "-DGGML_CUDA_MMV_Y=1", \- "-DGGML_MULTIPLATFORM", \- "-DGGML_CUDA_DMMV_X=32", \- "-DK_QUANTS_PER_ITERATION=2", \- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \- (FLAG_tinyblas \- ? "-DGGML_USE_TINYBLAS" \- : "-DGGML_USE_CUBLAS")+#define NVCC_FLAGS \+ "--shared", "--forward-unknown-to-host-compiler", "-use_fast_math", \+ "--compiler-options", \+ (!IsWindows() \+ ? (!IsAarch64() \+ ? "-fPIC -O3 -march=native -mtune=native" \+ : "-fPIC -O3 -march=native -mtune=native -ffixed-x28") \+ : "/nologo /EHsc /O2 /GR /MT"), \+ "-DNDEBUG", "-DGGML_BUILD=1", "-DGGML_SHARED=1", \+ "-DGGML_CUDA_MMV_Y=1", "-DGGML_MULTIPLATFORM", \+ "-DGGML_CUDA_DMMV_X=32", "-DK_QUANTS_PER_ITERATION=2", \+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \+ (FLAG_tinyblas ? "-DGGML_USE_TINYBLAS" : "-DGGML_USE_CUBLAS")static const struct Source {const char *zip;@@ -119,7 +112,8 @@ static struct Cuda {static const char *Dlerror(void) {const char *msg;msg = cosmo_dlerror();- if (!msg) msg = "null dlopen error";+ if (!msg)+ msg = "null dlopen error";return msg;}@@ -140,9 +134,7 @@ static bool FileExists(const char *path) {static bool IsExecutable(const char *path) {struct stat st;- return !stat(path, &st) &&- (st.st_mode & 0111) &&- !S_ISDIR(st.st_mode);+ return !stat(path, &st) && (st.st_mode & 0111) && !S_ISDIR(st.st_mode);}static bool CreateTempPath(const char *path, char tmp[static PATH_MAX]) {@@ -158,9 +150,7 @@ static bool CreateTempPath(const char *path, char tmp[static PATH_MAX]) {}}-static bool Compile(const char *src,- const char *tmp,- const char *out,+static bool Compile(const char *src, const char *tmp, const char *out,char *args[]) {int pid, ws;llamafile_log_command(args);@@ -178,7 +168,8 @@ static bool Compile(const char *src,}}if (ws) {- tinylog(__func__, ": warning: ", args[0], " returned nonzero exit status\n", NULL);+ tinylog(__func__, ": warning: ", args[0],+ " returned nonzero exit status\n", NULL);unlink(tmp);return false;}@@ -210,9 +201,11 @@ static bool get_rocm_bin_path(char path[static PATH_MAX], const char *bin) {// 2. use /opt/rocm/bin/$name if it existsconst char *hip_path = getenv("HIP_PATH");if (!hip_path) {- tinylog(__func__, ": note: $HIP_PATH/bin/", name, " does not exist\n", NULL);+ tinylog(__func__, ": note: $HIP_PATH/bin/", name, " does not exist\n",+ NULL);if (!FileExists((hip_path = "/opt/rocm"))) {- tinylog(__func__, ": note: /opt/rocm/bin/", name, " does not exist\n", NULL);+ tinylog(__func__, ": note: /opt/rocm/bin/", name,+ " does not exist\n", NULL);return false;}}@@ -234,7 +227,10 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {char hip_info_path[PATH_MAX];if (!get_rocm_bin_path(hip_info_path, "hipInfo") &&!get_rocm_bin_path(hip_info_path, "rocminfo")) {- tinylog(__func__, ": warning: can't find hipInfo/rocminfo commands for AMD GPU detection\n", NULL);+ tinylog(__func__,+ ": warning: can't find hipInfo/rocminfo commands for AMD GPU "+ "detection\n",+ NULL);return false;}@@ -271,49 +267,49 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {while ((rc = read(pipefds[0], buf, sizeof(buf))) > 0) {for (int i = 0; i < rc; ++i) {switch (t) {- case 0:- if (buf[i] == 'g') {- t = 1;- }- break;- case 1:- if (buf[i] == 'f') {- t = 2;- } else {- t = 0;- }- break;- case 2:- if (buf[i] == 'x') {- t = 3;- a = 0;- } else {- t = 0;- }- break;- case 3:- if (isdigit(buf[i])) {- a *= 10;- a += buf[i] - '0';- } else {- t = 0;- if ((a & 0xffff) && (a & 0xffff) == a) {- a &= 0xffff;- bool dupe = false;- for (int j = 0; j < 4; ++j) {- if (((archs >> (j * 16)) & 0xffff) == a) {- dupe = true;- }- }- if (!dupe) {- archs <<= 16;- archs |= a;+ case 0:+ if (buf[i] == 'g') {+ t = 1;+ }+ break;+ case 1:+ if (buf[i] == 'f') {+ t = 2;+ } else {+ t = 0;+ }+ break;+ case 2:+ if (buf[i] == 'x') {+ t = 3;+ a = 0;+ } else {+ t = 0;+ }+ break;+ case 3:+ if (isdigit(buf[i])) {+ a *= 10;+ a += buf[i] - '0';+ } else {+ t = 0;+ if ((a & 0xffff) && (a & 0xffff) == a) {+ a &= 0xffff;+ bool dupe = false;+ for (int j = 0; j < 4; ++j) {+ if (((archs >> (j * 16)) & 0xffff) == a) {+ dupe = true;}}+ if (!dupe) {+ archs <<= 16;+ archs |= a;+ }}- break;- default:- __builtin_unreachable();+ }+ break;+ default:+ __builtin_unreachable();}}}@@ -328,19 +324,23 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {}}if (ws) {- tinylog(__func__, ": error: hipInfo returned non-zero exit status\n", NULL);+ tinylog(__func__, ": error: hipInfo returned non-zero exit status\n",+ NULL);return false;}// Serialize value for --offload-arch=LIST flag.if (!archs) {- tinylog(__func__, ": warning: hipInfo output didn't list any graphics cards\n", NULL);+ tinylog(__func__,+ ": warning: hipInfo output didn't list any graphics cards\n",+ NULL);return false;}bool gotsome = false;char *p = stpcpy(out, "--offload-arch=");do {- if (gotsome) *p++ = ',';+ if (gotsome)+ *p++ = ',';p += sprintf(p, "gfx%d", archs & 0xffff);gotsome = true;} while ((archs >>= 16));@@ -366,15 +366,18 @@ static bool get_nvcc_path(char path[static PATH_MAX]) {}const char *cuda_path;if ((cuda_path = getenv("CUDA_PATH"))) {- if (!*cuda_path) return false;+ if (!*cuda_path)+ return false;strlcpy(path, cuda_path, PATH_MAX);strlcat(path, "/bin/", PATH_MAX);} else {- tinylog(__func__, ": note: $CUDA_PATH/bin/", name, " does not exist\n", NULL);+ tinylog(__func__, ": note: $CUDA_PATH/bin/", name, " does not exist\n",+ NULL);if (FileExists("/opt/cuda")) {strlcpy(path, "/opt/cuda/bin/", PATH_MAX);} else {- tinylog(__func__, ": note: /opt/cuda/bin/", name, " does not exist\n", NULL);+ tinylog(__func__, ": note: /opt/cuda/bin/", name,+ " does not exist\n", NULL);strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);}}@@ -387,7 +390,8 @@ static bool get_nvcc_path(char path[static PATH_MAX]) {}}-static dontinline bool get_nvcc_arch_flag(const char *nvcc, char flag[static 32]) {+static dontinline bool get_nvcc_arch_flag(const char *nvcc,+ char flag[static 32]) {// create path of exechar exe[PATH_MAX];@@ -416,7 +420,8 @@ static dontinline bool get_nvcc_arch_flag(const char *nvcc, char flag[static 32]// nvidia-smi in cuda 11.5+ can do this but (1) it takes longer to// run than compiling / running this script and (2) the nvidia-smi// command isn't available on Jetson devices.- tinylog(__func__, ": note: building nvidia compute capability detector...\n", NULL);+ tinylog(__func__,+ ": note: building nvidia compute capability detector...\n", NULL);if (!Compile(src, tmp, exe, (char *[]){(char *)nvcc, "-o", tmp, src, 0})) {return false;}@@ -455,14 +460,18 @@ static dontinline bool get_nvcc_arch_flag(const char *nvcc, char flag[static 32]}}if (ws) {- tinylog(__func__, ": error: compute capability detector returned nonzero exit status\n", NULL);+ tinylog(__func__,+ ": error: compute capability detector returned nonzero exit "+ "status\n",+ NULL);return false;}// parse output of detectorchar *endptr;if (!*ibuf || !strtol(ibuf, &endptr, 10) || *endptr) {- tinylog(__func__, ": error: bad compute capability detector output\n", NULL);+ tinylog(__func__, ": error: bad compute capability detector output\n",+ NULL);return false;}@@ -471,7 +480,8 @@ static dontinline bool get_nvcc_arch_flag(const char *nvcc, char flag[static 32]return true;}-static bool compile_amd_windows(const char *clangxx, const char *dso, const char *src, const char *tmpdso) {+static bool compile_amd_windows(const char *clangxx, const char *dso,+ const char *src, const char *tmpdso) {const char *lib = IsWindows() ? "lib" : GetDsoExtension();const char *hip_path = getenv("HIP_PATH");@@ -491,7 +501,8 @@ static bool compile_amd_windows(const char *clangxx, const char *dso, const char"-O3","-shared","-DNDEBUG",- "-x", "hip",+ "-x",+ "hip","--hip-link","-std=gnu++14","-fuse-ld=lld",@@ -505,22 +516,32 @@ static bool compile_amd_windows(const char *clangxx, const char *dso, const char"-Wno-ignored-attributes","-D_CRT_SECURE_NO_WARNINGS","-DK_QUANTS_PER_ITERATION=2",- "-o", (char *)tmpdso, (char *)src,- "-Xclang", "--dependent-lib=msvcrt",- "-mllvm", "-amdgpu-function-calls=false",- "-mllvm", "-amdgpu-early-inline-all=true",+ "-o",+ (char *)tmpdso,+ (char *)src,+ "-Xclang",+ "--dependent-lib=msvcrt",+ "-mllvm",+ "-amdgpu-function-calls=false",+ "-mllvm",+ "-amdgpu-early-inline-all=true",FLAG_tinyblas ? "-DGGML_USE_TINYBLAS" : "-DIGNORE",- "-isystem", gc(xasprintf("%s/include", hip_path)),- BLAS_ONLY("-l"), BLAS_ONLY(gc(xasprintf("%s/lib/hipblas.%s", hip_path, lib))),- BLAS_ONLY("-l"), BLAS_ONLY(gc(xasprintf("%s/lib/rocblas.%s", hip_path, lib))),- "-l", gc(xasprintf("%s/lib/amdhip64.%s", hip_path, lib)),+ "-isystem",+ gc(xasprintf("%s/include", hip_path)),+ BLAS_ONLY("-l"),+ BLAS_ONLY(gc(xasprintf("%s/lib/hipblas.%s", hip_path, lib))),+ BLAS_ONLY("-l"),+ BLAS_ONLY(gc(xasprintf("%s/lib/rocblas.%s", hip_path, lib))),+ "-l",+ gc(xasprintf("%s/lib/amdhip64.%s", hip_path, lib)),"-lkernel32",NULL,};return Compile(src, tmpdso, dso, args);}-static bool compile_amd_unix(const char *dso, const char *src, const char *tmpdso) {+static bool compile_amd_unix(const char *dso, const char *src,+ const char *tmpdso) {// get set of microarchitectures for all installed graphics cards// it's possible to safe --offload-arch=native but we do it ourself@@ -552,7 +573,9 @@ static bool compile_amd_unix(const char *dso, const char *src, const char *tmpds"-DK_QUANTS_PER_ITERATION=2","-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128",FLAG_tinyblas ? "-DGGML_USE_TINYBLAS" : "-DIGNORE",- "-o", (char *)tmpdso, (char *)src,+ "-o",+ (char *)tmpdso,+ (char *)src,BLAS_ONLY("-lhipblas"),BLAS_ONLY("-lrocblas"),NULL,@@ -584,10 +607,11 @@ static bool compile_nvidia(const char *nvcc, const char *dso, const char *src) {}// try building dso with host nvidia microarchitecture- tinylog(__func__, ": note: building ggml-cuda with nvcc -arch=native...\n", NULL);- if (Compile(src, tmpdso, dso, (char *[]){- (char *)nvcc, "-arch=native", NVCC_FLAGS, "-o", tmpdso,- (char *)src, NVCC_LIBS, NULL})) {+ tinylog(__func__, ": note: building ggml-cuda with nvcc -arch=native...\n",+ NULL);+ if (Compile(src, tmpdso, dso,+ (char *[]){(char *)nvcc, "-arch=native", NVCC_FLAGS, "-o",+ tmpdso, (char *)src, NVCC_LIBS, NULL})) {return true;}@@ -596,10 +620,11 @@ static bool compile_nvidia(const char *nvcc, const char *dso, const char *src) {if (!get_nvcc_arch_flag(nvcc, archflag)) {return false;}- tinylog(__func__, ": note: building ggml-cuda with nvcc ", archflag, "...\n", NULL);- if (Compile(src, tmpdso, dso, (char *[]){- (char *)nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,- (char *)src, NVCC_LIBS, NULL})) {+ tinylog(__func__, ": note: building ggml-cuda with nvcc ", archflag,+ "...\n", NULL);+ if (Compile(src, tmpdso, dso,+ (char *[]){(char *)nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,+ (char *)src, NVCC_LIBS, NULL})) {return true;}@@ -616,7 +641,8 @@ static bool extract_cuda_dso(const char *dso, const char *name) {strlcat(zip, ".", sizeof(zip));strlcat(zip, GetDsoExtension(), sizeof(zip));if (!FileExists(zip)) {- tinylog(__func__, ": note: prebuilt binary ", zip, " not found\n", NULL);+ tinylog(__func__, ": note: prebuilt binary ", zip, " not found\n",+ NULL);return false;}@@ -627,7 +653,8 @@ static bool extract_cuda_dso(const char *dso, const char *name) {static void *imp(void *lib, const char *sym) {void *fun = cosmo_dlsym(lib, sym);if (!fun) {- tinylog(__func__, ": error: failed to import symbol: ", sym, "\n", NULL);+ tinylog(__func__, ": error: failed to import symbol: ", sym, "\n",+ NULL);}return fun;}@@ -650,9 +677,11 @@ static bool link_cuda_dso(const char *dso, const char *dir) {}if (!lib) {char cc[PATH_MAX];- tinylog(__func__, ": warning: ", Dlerror(), ": failed to load library\n", NULL);+ tinylog(__func__, ": warning: ", Dlerror(),+ ": failed to load library\n", NULL);if ((IsLinux() || IsBsd()) && !commandv("cc", cc, PATH_MAX)) {- tinylog(__func__, ": note: you need to install cc for gpu support\n", NULL);+ tinylog(__func__,+ ": note: you need to install cc for gpu support\n", NULL);}return false;}@@ -666,17 +695,24 @@ static bool link_cuda_dso(const char *dso, const char *dir) {ok &= !!(ggml_cuda.host_malloc = imp(lib, "ggml_cuda_host_malloc"));ok &= !!(ggml_cuda.can_mul_mat = imp(lib, "ggml_cuda_can_mul_mat"));ok &= !!(ggml_cuda.compute_forward = imp(lib, "ggml_cuda_compute_forward"));- ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_cuda_get_device_count"));- ok &= !!(ggml_cuda.get_device_description = imp(lib, "ggml_cuda_get_device_description"));- ok &= !!(ggml_cuda.host_buffer_type = imp(lib, "ggml_backend_cuda_host_buffer_type"));+ ok &=+ !!(ggml_cuda.get_device_count = imp(lib, "ggml_cuda_get_device_count"));+ ok &= !!(ggml_cuda.get_device_description =+ imp(lib, "ggml_cuda_get_device_description"));+ ok &= !!(ggml_cuda.host_buffer_type =+ imp(lib, "ggml_backend_cuda_host_buffer_type"));ok &= !!(ggml_cuda.buffer_type = imp(lib, "ggml_backend_cuda_buffer_type"));ok &= !!(ggml_cuda.backend_init = imp(lib, "ggml_backend_cuda_init"));- ok &= !!(ggml_cuda.get_device_count2 = imp(lib, "ggml_backend_cuda_get_device_count"));- ok &= !!(ggml_cuda.split_buffer_type = imp(lib, "ggml_backend_cuda_split_buffer_type"));+ ok &= !!(ggml_cuda.get_device_count2 =+ imp(lib, "ggml_backend_cuda_get_device_count"));+ ok &= !!(ggml_cuda.split_buffer_type =+ imp(lib, "ggml_backend_cuda_split_buffer_type"));ok &= !!(ggml_cuda.reg_devices = imp(lib, "ggml_backend_cuda_reg_devices"));- ok &= !!(ggml_cuda.get_device_memory = imp(lib, "ggml_backend_cuda_get_device_memory"));+ ok &= !!(ggml_cuda.get_device_memory =+ imp(lib, "ggml_backend_cuda_get_device_memory"));if (!ok) {- tinylog(__func__, ": error: not all cuda symbols could be imported\n", NULL);+ tinylog(__func__, ": error: not all cuda symbols could be imported\n",+ NULL);cosmo_dlclose(lib);return false;}@@ -704,12 +740,12 @@ static bool import_cuda_impl(void) {// Check if we're allowed to even try.switch (FLAG_gpu) {- case LLAMAFILE_GPU_AUTO:- case LLAMAFILE_GPU_AMD:- case LLAMAFILE_GPU_NVIDIA:- break;- default:- return false;+ case LLAMAFILE_GPU_AUTO:+ case LLAMAFILE_GPU_AMD:+ case LLAMAFILE_GPU_NVIDIA:+ break;+ default:+ return false;}tinylog(__func__, ": initializing gpu module...\n", NULL);@@ -724,18 +760,18 @@ static bool import_cuda_impl(void) {}strlcat(src, srcs[i].name, sizeof(src));switch (llamafile_is_file_newer_than(srcs[i].zip, src)) {- case -1:+ case -1:+ return false;+ case false:+ break;+ case true:+ needs_rebuild = true;+ if (!llamafile_extract(srcs[i].zip, src)) {return false;- case false:- break;- case true:- needs_rebuild = true;- if (!llamafile_extract(srcs[i].zip, src)) {- return false;- }- break;- default:- __builtin_unreachable();+ }+ break;+ default:+ __builtin_unreachable();}}@@ -749,136 +785,137 @@ static bool import_cuda_impl(void) {// Attempt to load AMD GPU support.// We favor the underdog on AMD + NVIDIA chimeras.switch (FLAG_gpu) {- case LLAMAFILE_GPU_AMD:- case LLAMAFILE_GPU_AUTO:-- // Get some essential paths.- // ROCm SDK puts BLAS DLLs in same folder as clang++- if (get_rocm_bin_path(compiler_path_buf, "amdclang++") ||- get_rocm_bin_path(compiler_path_buf, "clang++")) {- strcpy(library_path_buf, compiler_path_buf);- dirname(library_path_buf);- compiler_path = compiler_path_buf;- library_path = library_path_buf;+ case LLAMAFILE_GPU_AMD:+ case LLAMAFILE_GPU_AUTO:++ // Get some essential paths.+ // ROCm SDK puts BLAS DLLs in same folder as clang+++ if (get_rocm_bin_path(compiler_path_buf, "amdclang++") ||+ get_rocm_bin_path(compiler_path_buf, "clang++")) {+ strcpy(library_path_buf, compiler_path_buf);+ dirname(library_path_buf);+ compiler_path = compiler_path_buf;+ library_path = library_path_buf;+ } else {+ compiler_path = 0;+ library_path = 0;+ }++ // Get path of GGML DSO for AMD.+ llamafile_get_app_dir(dso, PATH_MAX);+ strlcat(dso, "ggml-rocm.", PATH_MAX);+ strlcat(dso, GetDsoExtension(), PATH_MAX);+ if (FLAG_nocompile) {+ if ((FileExists(dso) || extract_cuda_dso(dso, "ggml-rocm")) &&+ link_cuda_dso(dso, library_path)) {+ return true;} else {- compiler_path = 0;- library_path = 0;+ goto TryNvidia;}+ }- // Get path of GGML DSO for AMD.- llamafile_get_app_dir(dso, PATH_MAX);- strlcat(dso, "ggml-rocm.", PATH_MAX);- strlcat(dso, GetDsoExtension(), PATH_MAX);- if (FLAG_nocompile) {- if ((FileExists(dso) ||- extract_cuda_dso(dso, "ggml-rocm")) &&- link_cuda_dso(dso, library_path)) {+ // Check if DSO is already compiled.+ if (!needs_rebuild && !FLAG_recompile) {+ switch (llamafile_is_file_newer_than(src, dso)) {+ case -1:+ return false;+ case false:+ if (link_cuda_dso(dso, library_path)) {return true;} else {goto TryNvidia;}+ case true:+ break;+ default:+ __builtin_unreachable();}+ }- // Check if DSO is already compiled.- if (!needs_rebuild && !FLAG_recompile) {- switch (llamafile_is_file_newer_than(src, dso)) {- case -1:- return false;- case false:- if (link_cuda_dso(dso, library_path)) {- return true;- } else {- goto TryNvidia;- }- case true:- break;- default:- __builtin_unreachable();- }- }-- // Try building CUDA with ROCm SDK.- if (compiler_path) {- if (compile_amd(compiler_path, dso, src)) {- if (link_cuda_dso(dso, library_path)) {- return true;- } else {- goto TryNvidia;- }- }- } else {- tinylog(__func__, ": won't compile AMD GPU support because $HIP_PATH/bin/clang++ is missing\n", NULL);- }-- // Try extracting prebuilt tinyBLAS DSO from PKZIP.- if (extract_cuda_dso(dso, "ggml-rocm")) {+ // Try building CUDA with ROCm SDK.+ if (compiler_path) {+ if (compile_amd(compiler_path, dso, src)) {if (link_cuda_dso(dso, library_path)) {return true;} else {goto TryNvidia;}}+ } else {+ tinylog(__func__,+ ": won't compile AMD GPU support because "+ "$HIP_PATH/bin/clang++ is missing\n",+ NULL);+ }++ // Try extracting prebuilt tinyBLAS DSO from PKZIP.+ if (extract_cuda_dso(dso, "ggml-rocm")) {+ if (link_cuda_dso(dso, library_path)) {+ return true;+ } else {+ goto TryNvidia;+ }+ }- break;- default:- break;+ break;+ default:+ break;}TryNvidia:// Attempt to load NVIDIA GPU support.switch (FLAG_gpu) {- case LLAMAFILE_GPU_AUTO:- case LLAMAFILE_GPU_NVIDIA:-- // Get some essential paths.- // CUDA SDK puts cuBLAS DLL in same folder as NVCC- if (get_nvcc_path(compiler_path_buf)) {- strcpy(library_path_buf, compiler_path_buf);- dirname(library_path_buf);- compiler_path = compiler_path_buf;- library_path = library_path_buf;- } else {- compiler_path = 0;- library_path = 0;- }-- // Get path of GGML DSO for NVIDIA.- llamafile_get_app_dir(dso, PATH_MAX);- strlcat(dso, "ggml-cuda.", PATH_MAX);- strlcat(dso, GetDsoExtension(), PATH_MAX);- if (FLAG_nocompile) {- return ((FileExists(dso) ||- extract_cuda_dso(dso, "ggml-cuda")) &&- link_cuda_dso(dso, library_path));- }+ case LLAMAFILE_GPU_AUTO:+ case LLAMAFILE_GPU_NVIDIA:++ // Get some essential paths.+ // CUDA SDK puts cuBLAS DLL in same folder as NVCC+ if (get_nvcc_path(compiler_path_buf)) {+ strcpy(library_path_buf, compiler_path_buf);+ dirname(library_path_buf);+ compiler_path = compiler_path_buf;+ library_path = library_path_buf;+ } else {+ compiler_path = 0;+ library_path = 0;+ }- // Check if DSO is already compiled.- if (!needs_rebuild && !FLAG_recompile) {- switch (llamafile_is_file_newer_than(src, dso)) {- case -1:- return false;- case false:- return link_cuda_dso(dso, library_path);- case true:- break;- default:- __builtin_unreachable();- }- }+ // Get path of GGML DSO for NVIDIA.+ llamafile_get_app_dir(dso, PATH_MAX);+ strlcat(dso, "ggml-cuda.", PATH_MAX);+ strlcat(dso, GetDsoExtension(), PATH_MAX);+ if (FLAG_nocompile) {+ return ((FileExists(dso) || extract_cuda_dso(dso, "ggml-cuda")) &&+ link_cuda_dso(dso, library_path));+ }- // Try building CUDA from source with mighty cuBLAS.- if (compiler_path && compile_nvidia(compiler_path, dso, src)) {+ // Check if DSO is already compiled.+ if (!needs_rebuild && !FLAG_recompile) {+ switch (llamafile_is_file_newer_than(src, dso)) {+ case -1:+ return false;+ case false:return link_cuda_dso(dso, library_path);+ case true:+ break;+ default:+ __builtin_unreachable();}+ }- // Try extracting prebuilt tinyBLAS DSO from PKZIP.- if (extract_cuda_dso(dso, "ggml-cuda")) {- return link_cuda_dso(dso, library_path);- }+ // Try building CUDA from source with mighty cuBLAS.+ if (compiler_path && compile_nvidia(compiler_path, dso, src)) {+ return link_cuda_dso(dso, library_path);+ }- break;- default:- break;+ // Try extracting prebuilt tinyBLAS DSO from PKZIP.+ if (extract_cuda_dso(dso, "ggml-cuda")) {+ return link_cuda_dso(dso, library_path);+ }++ break;+ default:+ break;}// too bad@@ -906,82 +943,98 @@ bool llamafile_has_cuda(void) {}GGML_CALL void ggml_init_cublas(void) {- if (!llamafile_has_cuda()) return;+ if (!llamafile_has_cuda())+ return;return ggml_cuda.init();}GGML_CALL bool ggml_cublas_loaded(void) {- if (!llamafile_has_cuda()) return false;+ if (!llamafile_has_cuda())+ return false;return ggml_cuda.loaded();}GGML_CALL void *ggml_cuda_host_malloc(size_t n) {- if (!llamafile_has_cuda()) return NULL;+ if (!llamafile_has_cuda())+ return NULL;return ggml_cuda.host_malloc(n);}GGML_CALL void ggml_cuda_host_free(void *data) {- if (!llamafile_has_cuda()) return;+ if (!llamafile_has_cuda())+ return;return ggml_cuda.host_free(data);}GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0,const struct ggml_tensor *src1,struct ggml_tensor *dst) {- if (!llamafile_has_cuda()) return false;+ if (!llamafile_has_cuda())+ return false;return ggml_cuda.can_mul_mat(src0, src1, dst);}GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params *params,struct ggml_tensor *tensor) {- if (!llamafile_has_cuda()) return false;+ if (!llamafile_has_cuda())+ return false;return ggml_cuda.compute_forward(params, tensor);}GGML_CALL int ggml_cuda_get_device_count(void) {- if (!llamafile_has_cuda()) return 0;+ if (!llamafile_has_cuda())+ return 0;return ggml_cuda.get_device_count();}-GGML_CALL void ggml_cuda_get_device_description(int device,- char *description,+GGML_CALL void ggml_cuda_get_device_description(int device, char *description,size_t description_size) {- if (!llamafile_has_cuda()) return;+ if (!llamafile_has_cuda())+ return;return ggml_cuda.get_device_description(device, description,description_size);}GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {- if (!llamafile_has_cuda()) return 0;+ if (!llamafile_has_cuda())+ return 0;return ggml_cuda.buffer_type(device);}GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {- if (!llamafile_has_cuda()) return 0;+ if (!llamafile_has_cuda())+ return 0;return ggml_cuda.host_buffer_type();}GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {- if (!llamafile_has_cuda()) return 0;+ if (!llamafile_has_cuda())+ return 0;return ggml_cuda.backend_init(device);}GGML_CALL int ggml_backend_cuda_get_device_count(void) {- if (!llamafile_has_cuda()) return 0;+ if (!llamafile_has_cuda())+ return 0;return ggml_cuda.get_device_count();}-GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float *tensor_split) {- if (!llamafile_has_cuda()) return 0;+GGML_CALL ggml_backend_buffer_type_t+ggml_backend_cuda_split_buffer_type(const float *tensor_split) {+ if (!llamafile_has_cuda())+ return 0;return ggml_cuda.split_buffer_type(tensor_split);}GGML_CALL int ggml_backend_cuda_reg_devices(void) {- if (!llamafile_has_cuda()) return 0;+ if (!llamafile_has_cuda())+ return 0;return ggml_cuda.reg_devices();}-GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) {- if (!llamafile_has_cuda()) return;+GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t *free,+ size_t *total) {+ if (!llamafile_has_cuda())+ return;return ggml_cuda.get_device_memory(device, free, total);}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 5d21d88f..71207b9c 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -1,7 +1,7 @@#ifndef LLAMAFILE_H_#define LLAMAFILE_H_-#include <stdio.h>#include <stdbool.h>+#include <stdio.h>#ifdef __cplusplusextern "C" {#endif@@ -29,6 +29,8 @@ void llamafile_schlep(const void *, size_t);void llamafile_get_app_dir(char *, size_t);void llamafile_launch_browser(const char *);+extern bool FLAG_unsecure;+#define LLAMAFILE_GPU_ERROR -2#define LLAMAFILE_GPU_DISABLE -1#define LLAMAFILE_GPU_AUTO 0@@ -46,8 +48,8 @@ bool llamafile_has_metal(void);int llamafile_gpu_parse(const char *);const char *llamafile_describe_gpu(void);-void llamafile_sgemm(long, long, long, int, const void *, long,- const void *, long, float *, long, long, long);+void llamafile_sgemm(long, long, long, int, const void *, long, const void *,+ long, float *, long, long, long);#ifdef __cplusplus}diff --git a/llamafile/security.c b/llamafile/security.cnew file mode 100644index 00000000..84781bff--- /dev/null+++ b/llamafile/security.c@@ -0,0 +1,20 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llamafile.h"++bool FLAG_unsecure;diff --git a/llamafile/simple.cpp b/llamafile/simple.cppindex d7211dea..e505ad54 100644--- a/llamafile/simple.cpp+++ b/llamafile/simple.cpp@@ -1,4 +1,4 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi//// Copyright 2024 Mozilla Foundation@@ -16,24 +16,26 @@// limitations under the License.#include <cmath>+#include <cosmo.h>#include <cstdio>#include <string>#include <vector>-#include <cosmo.h>-#include "llama.cpp/llama.h"#include "llama.cpp/common.h"+#include "llama.cpp/llama.h"#include "llamafile/llamafile.h"static bool eval_tokens(struct llama_context *ctx_llama,- std::vector<llama_token> tokens,- int n_batch, int *n_past) {+ std::vector<llama_token> tokens, int n_batch,+ int *n_past) {int N = (int)tokens.size();for (int i = 0; i < N; i += n_batch) {int n_eval = (int)tokens.size() - i;- if (n_eval > n_batch) n_eval = n_batch;- if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0))) {- return false; // probably ran out of context+ if (n_eval > n_batch)+ n_eval = n_batch;+ if (llama_decode(ctx_llama,+ llama_batch_get_one(&tokens[i], n_eval, *n_past, 0))) {+ return false; // probably ran out of context}*n_past += n_eval;}@@ -49,7 +51,8 @@ static bool eval_id(struct llama_context *ctx_llama, int id, int *n_past) {static bool eval_string(struct llama_context *ctx_llama, const char *str,int n_batch, int *n_past, bool add_bos) {std::string str2 = str;- std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos);+ std::vector<llama_token> embd_inp =+ ::llama_tokenize(ctx_llama, str2, add_bos);return eval_tokens(ctx_llama, embd_inp, n_batch, n_past);}@@ -57,30 +60,46 @@ int main(int argc, char **argv) {gpt_params params;params.n_ctx = 0;FLAG_log_disable = true;- if (!gpt_params_parse(argc, argv, params)) return 1;- if (params.prompt.empty()) params.prompt = "The";- llama_backend_init(params.numa);++ if (!gpt_params_parse(argc, argv, params))+ return 1;++ if (params.prompt.empty())+ params.prompt = "The";++ llama_backend_init();+llama_model_params model_params = llama_model_default_params();model_params.n_gpu_layers = llamafile_gpu_layers(35);- llama_model *model = llama_load_model_from_file(params.model.c_str(), model_params);- if (model == NULL) return 2;- llama_context_params ctx_params = llama_context_params_from_gpt_params(params);+ llama_model *model =+ llama_load_model_from_file(params.model.c_str(), model_params);+ if (model == NULL)+ return 2;++ llama_context_params ctx_params =+ llama_context_params_from_gpt_params(params);llama_context *ctx = llama_new_context_with_model(model, ctx_params);- if (ctx == NULL) return 3;+ if (ctx == NULL)+ return 3;+printf("%s", params.prompt.c_str());int n_past = 0;bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));eval_string(ctx, params.prompt.c_str(), params.n_batch, &n_past, add_bos);- struct llama_sampling_context *ctx_sampling = llama_sampling_init(params.sparams);+ struct llama_sampling_context *ctx_sampling =+ llama_sampling_init(params.sparams);for (;;) {llama_token id = llama_sampling_sample(ctx_sampling, ctx, NULL);llama_sampling_accept(ctx_sampling, ctx, id, true);- if (id == llama_token_eos(model)) break;+ if (id == llama_token_eos(model))+ break;printf("%s", llama_token_to_piece(ctx, id).c_str());fflush(stdout);- if (!eval_id(ctx, id, &n_past)) break;+ if (!eval_id(ctx, id, &n_past))+ break;}printf("\n");+llama_sampling_free(ctx_sampling);llama_free(ctx);llama_free_model(model);