Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Synchronize with llama.cpp
Justine Tunneyc0208c155cea
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex 02f1e61b..a4f3ec5f 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -9,8 +9,8 @@ LICENSEORIGINhttps://github.com/ggerganov/llama.cpp/pull/4406/- cb49e0f8c906e5da49e9f6d64a57742a9a241c6a- 2024-02-27+ fa046eafbc70bf97dcf39843af0323f19a8c9ac3+ 2024-03-22LOCAL MODIFICATIONSdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 02d42821..19acff7d 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -48,6 +48,8 @@#pragma warning(disable: 4244 4267) // possible loss of data#endif+bool passed_gpu_flags;+int32_t get_num_physical_cores() {#ifdef __linux__// enumerate the set of thread siblings, num entries is num cores@@ -140,765 +142,1075 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {return result;}-bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {- bool invalid_param = false;- std::string arg;- bool passed_gpu_flags = false;- const std::string arg_prefix = "--";- llama_sampling_params & sparams = params.sparams;+static bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param) {+ llama_sampling_params& sparams = params.sparams;- for (int i = 1; i < argc; i++) {- arg = argv[i];- if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {- std::replace(arg.begin(), arg.end(), '_', '-');+ if (arg == "--cli") {+ return true;+ }+ if (arg == "--unsecure") {+ FLAG_unsecure = true;+ return true;+ }+ if (arg == "--nocompile") {+ FLAG_nocompile = true;+ return true;+ }+ if (arg == "--recompile") {+ FLAG_recompile = true;+ return true;+ }+ if (arg == "--tinyblas") {+ FLAG_tinyblas = true; // undocumented+ return true;+ }+ if (arg == "--gpu") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ FLAG_gpu = llamafile_gpu_parse(argv[i]);+ if (FLAG_gpu == LLAMAFILE_GPU_ERROR) {+ fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);+ exit(1);}+ return true;+ }- if (arg == "--cli") {- // do nothing- } else if (arg == "-s" || arg == "--seed") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.seed = std::stoul(argv[i]);- } else if (arg == "-t" || arg == "--threads") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_threads = std::stoi(argv[i]);- if (params.n_threads <= 0) {- params.n_threads = std::thread::hardware_concurrency();- }- } else if (arg == "-tb" || arg == "--threads-batch") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_threads_batch = std::stoi(argv[i]);- if (params.n_threads_batch <= 0) {- params.n_threads_batch = std::thread::hardware_concurrency();- }- } else if (arg == "-td" || arg == "--threads-draft") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_threads_draft = std::stoi(argv[i]);- if (params.n_threads_draft <= 0) {- params.n_threads_draft = std::thread::hardware_concurrency();- }- } else if (arg == "-tbd" || arg == "--threads-batch-draft") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_threads_batch_draft = std::stoi(argv[i]);- if (params.n_threads_batch_draft <= 0) {- params.n_threads_batch_draft = std::thread::hardware_concurrency();- }- } else if (arg == "-p" || arg == "--prompt") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.prompt = argv[i];- } else if (arg == "-e" || arg == "--escape") {- params.escape = true;- } else if (arg == "--prompt-cache") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.path_prompt_cache = argv[i];- } else if (arg == "--prompt-cache-all") {- params.prompt_cache_all = true;- } else if (arg == "--prompt-cache-ro") {- params.prompt_cache_ro = true;- } else if (arg == "-bf" || arg == "--binary-file") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::ifstream file(argv[i], std::ios::binary);- if (!file) {- fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);- invalid_param = true;- break;- }- // store the external file name in params- params.prompt_file = argv[i];- std::ostringstream ss;- ss << file.rdbuf();- params.prompt = ss.str();- fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), argv[i]);- } else if (arg == "-f" || arg == "--file") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::ifstream file(argv[i]);- if (!file) {- fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);- invalid_param = true;- break;- }- // store the external file name in params- params.prompt_file = argv[i];- std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));- if (!params.prompt.empty() && params.prompt.back() == '\n') {- params.prompt.pop_back();- }- } else if (arg == "-n" || arg == "--n-predict") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_predict = std::stoi(argv[i]);- } else if (arg == "--top-k") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.top_k = std::stoi(argv[i]);- } else if (arg == "-c" || arg == "--ctx-size") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_ctx = std::stoi(argv[i]);- } else if (arg == "--grp-attn-n" || arg == "-gan") {- if (++i >= argc) {- invalid_param = true;- break;+ if (arg == "-s" || arg == "--seed") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.seed = std::stoul(argv[i]);+ return true;+ }+ if (arg == "-t" || arg == "--threads") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_threads = std::stoi(argv[i]);+ if (params.n_threads <= 0) {+ params.n_threads = std::thread::hardware_concurrency();+ }+ return true;+ }+ if (arg == "-tb" || arg == "--threads-batch") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_threads_batch = std::stoi(argv[i]);+ if (params.n_threads_batch <= 0) {+ params.n_threads_batch = std::thread::hardware_concurrency();+ }+ return true;+ }+ if (arg == "-td" || arg == "--threads-draft") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_threads_draft = std::stoi(argv[i]);+ if (params.n_threads_draft <= 0) {+ params.n_threads_draft = std::thread::hardware_concurrency();+ }+ return true;+ }+ if (arg == "-tbd" || arg == "--threads-batch-draft") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_threads_batch_draft = std::stoi(argv[i]);+ if (params.n_threads_batch_draft <= 0) {+ params.n_threads_batch_draft = std::thread::hardware_concurrency();+ }+ return true;+ }+ if (arg == "-p" || arg == "--prompt") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.prompt = argv[i];+ return true;+ }+ if (arg == "-e" || arg == "--escape") {+ params.escape = true;+ return true;+ }+ if (arg == "--prompt-cache") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.path_prompt_cache = argv[i];+ return true;+ }+ if (arg == "--prompt-cache-all") {+ params.prompt_cache_all = true;+ return true;+ }+ if (arg == "--prompt-cache-ro") {+ params.prompt_cache_ro = true;+ return true;+ }+ if (arg == "-bf" || arg == "--binary-file") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::ifstream file(argv[i], std::ios::binary);+ if (!file) {+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);+ invalid_param = true;+ return true;+ }+ // store the external file name in params+ params.prompt_file = argv[i];+ std::ostringstream ss;+ ss << file.rdbuf();+ params.prompt = ss.str();+ fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), argv[i]);+ return true;+ }+ if (arg == "-f" || arg == "--file") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::ifstream file(argv[i]);+ if (!file) {+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);+ invalid_param = true;+ return true;+ }+ // store the external file name in params+ params.prompt_file = argv[i];+ std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));+ if (!params.prompt.empty() && params.prompt.back() == '\n') {+ params.prompt.pop_back();+ }+ return true;+ }+ if (arg == "-n" || arg == "--n-predict") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_predict = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--top-k") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.top_k = std::stoi(argv[i]);+ return true;+ }+ if (arg == "-c" || arg == "--ctx-size") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_ctx = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--grp-attn-n" || arg == "-gan") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.grp_attn_n = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--grp-attn-w" || arg == "-gaw") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.grp_attn_w = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--rope-freq-base") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.rope_freq_base = std::stof(argv[i]);+ return true;+ }+ if (arg == "--rope-freq-scale") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.rope_freq_scale = std::stof(argv[i]);+ return true;+ }+ if (arg == "--rope-scaling") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::string value(argv[i]);+ /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; }+ else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }+ else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }+ else { invalid_param = true; }+ return true;+ }+ if (arg == "--rope-scale") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.rope_freq_scale = 1.0f / std::stof(argv[i]);+ return true;+ }+ if (arg == "--yarn-orig-ctx") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.yarn_orig_ctx = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--yarn-ext-factor") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.yarn_ext_factor = std::stof(argv[i]);+ return true;+ }+ if (arg == "--yarn-attn-factor") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.yarn_attn_factor = std::stof(argv[i]);+ return true;+ }+ if (arg == "--yarn-beta-fast") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.yarn_beta_fast = std::stof(argv[i]);+ return true;+ }+ if (arg == "--yarn-beta-slow") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.yarn_beta_slow = std::stof(argv[i]);+ return true;+ }+ if (arg == "--pooling") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::string value(argv[i]);+ /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }+ else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }+ else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }+ else { invalid_param = true; }+ return true;+ }+ if (arg == "--defrag-thold" || arg == "-dt") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.defrag_thold = std::stof(argv[i]);+ return true;+ }+ if (arg == "--samplers") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ const auto sampler_names = string_split(argv[i], ';');+ sparams.samplers_sequence = sampler_types_from_names(sampler_names, true);+ return true;+ }+ if (arg == "--sampling-seq") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.samplers_sequence = sampler_types_from_chars(argv[i]);+ return true;+ }+ if (arg == "--top-p") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.top_p = std::stof(argv[i]);+ return true;+ }+ if (arg == "--min-p") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.min_p = std::stof(argv[i]);+ return true;+ }+ if (arg == "--temp") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.temp = std::stof(argv[i]);+ sparams.temp = std::max(sparams.temp, 0.0f);+ return true;+ }+ if (arg == "--tfs") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.tfs_z = std::stof(argv[i]);+ return true;+ }+ if (arg == "--typical") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.typical_p = std::stof(argv[i]);+ return true;+ }+ if (arg == "--repeat-last-n") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.penalty_last_n = std::stoi(argv[i]);+ sparams.n_prev = std::max(sparams.n_prev, sparams.penalty_last_n);+ return true;+ }+ if (arg == "--repeat-penalty") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.penalty_repeat = std::stof(argv[i]);+ return true;+ }+ if (arg == "--frequency-penalty") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.penalty_freq = std::stof(argv[i]);+ return true;+ }+ if (arg == "--presence-penalty") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.penalty_present = std::stof(argv[i]);+ return true;+ }+ if (arg == "--dynatemp-range") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.dynatemp_range = std::stof(argv[i]);+ return true;+ }+ if (arg == "--dynatemp-exp") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.dynatemp_exponent = std::stof(argv[i]);+ return true;+ }+ if (arg == "--mirostat") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.mirostat = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--mirostat-lr") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.mirostat_eta = std::stof(argv[i]);+ return true;+ }+ if (arg == "--mirostat-ent") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.mirostat_tau = std::stof(argv[i]);+ return true;+ }+ if (arg == "--cfg-negative-prompt") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.cfg_negative_prompt = argv[i];+ return true;+ }+ if (arg == "--cfg-negative-prompt-file") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::ifstream file(argv[i]);+ if (!file) {+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);+ invalid_param = true;+ return true;+ }+ std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(sparams.cfg_negative_prompt));+ if (!sparams.cfg_negative_prompt.empty() && sparams.cfg_negative_prompt.back() == '\n') {+ sparams.cfg_negative_prompt.pop_back();+ }+ return true;+ }+ if (arg == "--cfg-scale") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.cfg_scale = std::stof(argv[i]);+ return true;+ }+ if (arg == "-b" || arg == "--batch-size") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_batch = std::stoi(argv[i]);+ return true;+ }+ if (arg == "-ub" || arg == "--ubatch-size") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_ubatch = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--keep") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_keep = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--draft") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_draft = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--chunks") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_chunks = std::stoi(argv[i]);+ return true;+ }+ if (arg == "-np" || arg == "--parallel") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_parallel = std::stoi(argv[i]);+ return true;+ }+ if (arg == "-ns" || arg == "--sequences") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_sequences = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--p-split" || arg == "-ps") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.p_split = std::stof(argv[i]);+ return true;+ }+ if (arg == "-m" || arg == "--model") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.model = argv[i];+ return true;+ }+ if (arg == "-mu" || arg == "--model-url") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.model_url = argv[i];+ return true;+ }+ if (arg == "-md" || arg == "--model-draft") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.model_draft = argv[i];+ return true;+ }+ if (arg == "-a" || arg == "--alias") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.model_alias = argv[i];+ return true;+ }+ if (arg == "--lora") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.lora_adapter.emplace_back(argv[i], 1.0f);+ params.use_mmap = false;+ return true;+ }+ if (arg == "--lora-scaled") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ const char* lora_adapter = argv[i];+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));+ params.use_mmap = false;+ return true;+ }+ if (arg == "--lora-base") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.lora_base = argv[i];+ return true;+ }+ if (arg == "--control-vector") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.control_vectors.push_back({ 1.0f, argv[i], });+ return true;+ }+ if (arg == "--control-vector-scaled") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ const char* fname = argv[i];+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.control_vectors.push_back({ std::stof(argv[i]), fname, });+ return true;+ }+ if (arg == "--control-vector-layer-range") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.control_vector_layer_start = std::stoi(argv[i]);+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.control_vector_layer_end = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--mmproj") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.mmproj = argv[i];+ return true;+ }+ if (arg == "--image") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.image = argv[i];+ return true;+ }+ if (arg == "-i" || arg == "--interactive") {+ params.interactive = true;+ return true;+ }+ if (arg == "--embedding") {+ params.embedding = true;+ return true;+ }+ if (arg == "--interactive-first") {+ params.interactive_first = true;+ return true;+ }+ if (arg == "-ins" || arg == "--instruct") {+ params.instruct = true;+ return true;+ }+ if (arg == "-cml" || arg == "--chatml") {+ params.chatml = true;+ return true;+ }+ if (arg == "--infill") {+ params.infill = true;+ return true;+ }+ if (arg == "-dkvc" || arg == "--dump-kv-cache") {+ params.dump_kv_cache = true;+ return true;+ }+ if (arg == "-nkvo" || arg == "--no-kv-offload") {+ params.no_kv_offload = true;+ return true;+ }+ if (arg == "-ctk" || arg == "--cache-type-k") {+ params.cache_type_k = argv[++i];+ return true;+ }+ if (arg == "-ctv" || arg == "--cache-type-v") {+ params.cache_type_v = argv[++i];+ return true;+ }+ if (arg == "--multiline-input") {+ params.multiline_input = true;+ return true;+ }+ if (arg == "--simple-io") {+ params.simple_io = true;+ return true;+ }+ if (arg == "-cb" || arg == "--cont-batching") {+ params.cont_batching = true;+ return true;+ }+ if (arg == "--color") {+ params.use_color = true;+ return true;+ }+ if (arg == "--mlock") {+ params.use_mlock = true;+ return true;+ }+ if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {+ passed_gpu_flags = true;+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_gpu_layers = std::stoi(argv[i]);+ if (params.n_gpu_layers <= 0)+ FLAG_gpu = LLAMAFILE_GPU_DISABLE;+ return true;+ }+ if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") {+ passed_gpu_flags = true;+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_gpu_layers_draft = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--main-gpu" || arg == "-mg") {+ passed_gpu_flags = true;+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.main_gpu = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--split-mode" || arg == "-sm") {+ passed_gpu_flags = true;+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::string arg_next = argv[i];+ if (arg_next == "none") {+ params.split_mode = LLAMA_SPLIT_MODE_NONE;+ }+ else if (arg_next == "layer") {+ params.split_mode = LLAMA_SPLIT_MODE_LAYER;+ }+ else if (arg_next == "row") {+#ifdef GGML_USE_SYCL+ fprintf(stderr, "warning: The split mode value:[row] is not supported by llama.cpp with SYCL. It's developing.\nExit!\n");+ exit(1);+#endif // GGML_USE_SYCL+ params.split_mode = LLAMA_SPLIT_MODE_ROW;+ }+ else {+ invalid_param = true;+ return true;+ }+ return true;+ }+ if (arg == "--tensor-split" || arg == "-ts") {+ passed_gpu_flags = true;+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::string arg_next = argv[i];++ // split string by , and /+ const std::regex regex{ R"([,/]+)" };+ std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };+ std::vector<std::string> split_arg{ it, {} };+ if (split_arg.size() >= llama_max_devices()) {+ invalid_param = true;+ return true;+ }+ for (size_t i = 0; i < llama_max_devices(); ++i) {+ if (i < split_arg.size()) {+ params.tensor_split[i] = std::stof(split_arg[i]);}-- params.grp_attn_n = std::stoi(argv[i]);- } else if (arg == "--grp-attn-w" || arg == "-gaw") {- if (++i >= argc) {- invalid_param = true;- break;+ else {+ params.tensor_split[i] = 0.0f;}+ }+ return true;+ }+ if (arg == "--no-mmap") {+ params.use_mmap = false;+ return true;+ }+ if (arg == "--numa") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::string value(argv[i]);+ /**/ if (value == "distribute" || value == "") { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }+ else { invalid_param = true; }+ return true;+ }+ if (arg == "--verbose-prompt") {+ params.verbose_prompt = true;+ return true;+ }+ if (arg == "--no-display-prompt") {+ params.display_prompt = false;+ return true;+ }+ if (arg == "-r" || arg == "--reverse-prompt") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.antiprompt.emplace_back(argv[i]);+ return true;+ }+ if (arg == "-ld" || arg == "--logdir") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.logdir = argv[i];- params.grp_attn_w = std::stoi(argv[i]);- } else if (arg == "--rope-freq-base") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.rope_freq_base = std::stof(argv[i]);- } else if (arg == "--rope-freq-scale") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.rope_freq_scale = std::stof(argv[i]);- } else if (arg == "--rope-scaling") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::string value(argv[i]);- /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; }- else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }- else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }- else { invalid_param = true; break; }- } else if (arg == "--rope-scale") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.rope_freq_scale = 1.0f/std::stof(argv[i]);- } else if (arg == "--yarn-orig-ctx") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.yarn_orig_ctx = std::stoi(argv[i]);- } else if (arg == "--yarn-ext-factor") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.yarn_ext_factor = std::stof(argv[i]);- } else if (arg == "--yarn-attn-factor") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.yarn_attn_factor = std::stof(argv[i]);- } else if (arg == "--yarn-beta-fast") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.yarn_beta_fast = std::stof(argv[i]);- } else if (arg == "--yarn-beta-slow") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.yarn_beta_slow = std::stof(argv[i]);- } else if (arg == "--defrag-thold" || arg == "-dt") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.defrag_thold = std::stof(argv[i]);- } else if (arg == "--samplers") {- if (++i >= argc) {- invalid_param = true;- break;- }- const auto sampler_names = string_split(argv[i], ';');- sparams.samplers_sequence = sampler_types_from_names(sampler_names, true);- } else if (arg == "--sampling-seq") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.samplers_sequence = sampler_types_from_chars(argv[i]);- } else if (arg == "--top-p") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.top_p = std::stof(argv[i]);- } else if (arg == "--min-p") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.min_p = std::stof(argv[i]);- } else if (arg == "--temp") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.temp = std::stof(argv[i]);- sparams.temp = std::max(sparams.temp, 0.0f);- } else if (arg == "--tfs") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.tfs_z = std::stof(argv[i]);- } else if (arg == "--typical") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.typical_p = std::stof(argv[i]);- } else if (arg == "--repeat-last-n") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.penalty_last_n = std::stoi(argv[i]);- sparams.n_prev = std::max(sparams.n_prev, sparams.penalty_last_n);- } else if (arg == "--repeat-penalty") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.penalty_repeat = std::stof(argv[i]);- } else if (arg == "--frequency-penalty") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.penalty_freq = std::stof(argv[i]);- } else if (arg == "--presence-penalty") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.penalty_present = std::stof(argv[i]);- } else if (arg == "--dynatemp-range") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.dynatemp_range = std::stof(argv[i]);- } else if (arg == "--dynatemp-exp") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.dynatemp_exponent = std::stof(argv[i]);- } else if (arg == "--mirostat") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.mirostat = std::stoi(argv[i]);- } else if (arg == "--mirostat-lr") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.mirostat_eta = std::stof(argv[i]);- } else if (arg == "--mirostat-ent") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.mirostat_tau = std::stof(argv[i]);- } else if (arg == "--cfg-negative-prompt") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.cfg_negative_prompt = argv[i];- } else if (arg == "--cfg-negative-prompt-file") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::ifstream file(argv[i]);- if (!file) {- fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);- invalid_param = true;- break;- }- std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(sparams.cfg_negative_prompt));- if (!sparams.cfg_negative_prompt.empty() && sparams.cfg_negative_prompt.back() == '\n') {- sparams.cfg_negative_prompt.pop_back();- }- } else if (arg == "--cfg-scale") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.cfg_scale = std::stof(argv[i]);- } else if (arg == "-b" || arg == "--batch-size") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_batch = std::stoi(argv[i]);- } else if (arg == "--keep") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_keep = std::stoi(argv[i]);- } else if (arg == "--draft") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_draft = std::stoi(argv[i]);- } else if (arg == "--chunks") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_chunks = std::stoi(argv[i]);- } else if (arg == "-np" || arg == "--parallel") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_parallel = std::stoi(argv[i]);- } else if (arg == "-ns" || arg == "--sequences") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_sequences = std::stoi(argv[i]);- } else if (arg == "--p-accept" || arg == "-pa") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.p_accept = std::stof(argv[i]);- } else if (arg == "--p-split" || arg == "-ps") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.p_split = std::stof(argv[i]);- } else if (arg == "-m" || arg == "--model") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.model = argv[i];- } else if (arg == "-md" || arg == "--model-draft") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.model_draft = argv[i];- } else if (arg == "-a" || arg == "--alias") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.model_alias = argv[i];- } else if (arg == "--lora") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.lora_adapter.emplace_back(argv[i], 1.0f);- params.use_mmap = false;- } else if (arg == "--lora-scaled") {- if (++i >= argc) {- invalid_param = true;- break;- }- const char * lora_adapter = argv[i];- if (++i >= argc) {- invalid_param = true;- break;- }- params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));- params.use_mmap = false;- } else if (arg == "--lora-base") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.lora_base = argv[i];- } else if (arg == "--mmproj") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.mmproj = argv[i];- } else if (arg == "--image") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.image = argv[i];- } else if (arg == "-i" || arg == "--interactive") {- params.interactive = true;- } else if (arg == "--embedding") {- params.embedding = true;- } else if (arg == "--interactive-first") {- params.interactive_first = true;- } else if (arg == "-ins" || arg == "--instruct") {- params.instruct = true;- } else if (arg == "-cml" || arg == "--chatml") {- params.chatml = true;- } else if (arg == "--infill") {- params.infill = true;- } else if (arg == "--unsecure") {- FLAG_unsecure = true;- } else if (arg == "--nocompile") {- FLAG_nocompile = true;- } else if (arg == "--recompile") {- FLAG_recompile = true;- } else if (arg == "--tinyblas") {- FLAG_tinyblas = true; // undocumented- } else if (arg == "--gpu") {- if (++i >= argc) {- invalid_param = true;- break;- }- FLAG_gpu = llamafile_gpu_parse(argv[i]);- if (FLAG_gpu == LLAMAFILE_GPU_ERROR) {- fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);- exit(1);- }- } else if (arg == "-dkvc" || arg == "--dump-kv-cache") {- params.dump_kv_cache = true;- } else if (arg == "-nkvo" || arg == "--no-kv-offload") {- params.no_kv_offload = true;- } else if (arg == "-ctk" || arg == "--cache-type-k") {- params.cache_type_k = argv[++i];- } else if (arg == "-ctv" || arg == "--cache-type-v") {- params.cache_type_v = argv[++i];- } else if (arg == "--multiline-input") {- params.multiline_input = true;- } else if (arg == "--simple-io") {- params.simple_io = true;- } else if (arg == "-cb" || arg == "--cont-batching") {- params.cont_batching = true;- } else if (arg == "--color") {- params.use_color = true;- } else if (arg == "--mlock") {- params.use_mlock = true;- } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {- passed_gpu_flags = true;- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_gpu_layers = std::stoi(argv[i]);- if (params.n_gpu_layers <= 0) {- FLAG_gpu = LLAMAFILE_GPU_DISABLE;+ if (params.logdir.back() != DIRECTORY_SEPARATOR) {+ params.logdir += DIRECTORY_SEPARATOR;+ }+ return true;+ }+ if (arg == "--save-all-logits" || arg == "--kl-divergence-base") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.logits_file = argv[i];+ return true;+ }+ if (arg == "--perplexity" || arg == "--all-logits") {+ params.logits_all = true;+ return true;+ }+ if (arg == "--ppl-stride") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.ppl_stride = std::stoi(argv[i]);+ return true;+ }+ if (arg == "-ptc" || arg == "--print-token-count") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.n_print = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--ppl-output-type") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.ppl_output_type = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--hellaswag") {+ params.hellaswag = true;+ return true;+ }+ if (arg == "--hellaswag-tasks") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.hellaswag_tasks = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--winogrande") {+ params.winogrande = true;+ return true;+ }+ if (arg == "--winogrande-tasks") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.winogrande_tasks = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--multiple-choice") {+ params.multiple_choice = true;+ return true;+ }+ if (arg == "--multiple-choice-tasks") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.multiple_choice_tasks = std::stoi(argv[i]);+ return true;+ }+ if (arg == "--kl-divergence") {+ params.kl_divergence = true;+ return true;+ }+ if (arg == "--ignore-eos") {+ params.ignore_eos = true;+ return true;+ }+ if (arg == "--no-penalize-nl") {+ sparams.penalize_nl = false;+ return true;+ }+ if (arg == "-l" || arg == "--logit-bias") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::stringstream ss(argv[i]);+ llama_token key;+ char sign;+ std::string value_str;+ try {+ if (ss >> key && ss >> sign && std::getline(ss, value_str) && (sign == '+' || sign == '-')) {+ sparams.logit_bias[key] = std::stof(value_str) * ((sign == '-') ? -1.0f : 1.0f);}- } else if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") {- passed_gpu_flags = true;- if (++i >= argc) {- invalid_param = true;- break;+ else {+ throw std::exception();}- params.n_gpu_layers_draft = std::stoi(argv[i]);- } else if (arg == "--main-gpu" || arg == "-mg") {- passed_gpu_flags = true;- if (++i >= argc) {- invalid_param = true;- break;+ }+ catch (const std::exception&) {+ invalid_param = true;+ return true;+ }+ return true;+ }+ if (arg == "-h" || arg == "--help") {+ gpt_print_usage(argc, argv, gpt_params());+ exit(0);+ }+ if (arg == "--version") {+ fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);+ fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET);+ exit(0);+ }+ if (arg == "--random-prompt") {+ params.random_prompt = true;+ return true;+ }+ if (arg == "--in-prefix-bos") {+ params.input_prefix_bos = true;+ return true;+ }+ if (arg == "--in-prefix") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.input_prefix = argv[i];+ return true;+ }+ if (arg == "--in-suffix") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.input_suffix = argv[i];+ return true;+ }+ if (arg == "--grammar") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ sparams.grammar = argv[i];+ return true;+ }+ if (arg == "--grammar-file") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ std::ifstream file(argv[i]);+ if (!file) {+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);+ invalid_param = true;+ return true;+ }+ std::copy(+ std::istreambuf_iterator<char>(file),+ std::istreambuf_iterator<char>(),+ std::back_inserter(sparams.grammar)+ );+ return true;+ }+ if (arg == "--override-kv") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ char* sep = strchr(argv[i], '=');+ if (sep == nullptr || sep - argv[i] >= 128) {+ fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);+ invalid_param = true;+ return true;+ }+ struct llama_model_kv_override kvo;+ std::strncpy(kvo.key, argv[i], sep - argv[i]);+ kvo.key[sep - argv[i]] = 0;+ sep++;+ if (strncmp(sep, "int:", 4) == 0) {+ sep += 4;+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;+ kvo.int_value = std::atol(sep);+ }+ else if (strncmp(sep, "float:", 6) == 0) {+ sep += 6;+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;+ kvo.float_value = std::atof(sep);+ }+ else if (strncmp(sep, "bool:", 5) == 0) {+ sep += 5;+ kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;+ if (std::strcmp(sep, "true") == 0) {+ kvo.bool_value = true;}- params.main_gpu = std::stoi(argv[i]);- } else if (arg == "--split-mode" || arg == "-sm") {- if (++i >= argc) {- invalid_param = true;- break;+ else if (std::strcmp(sep, "false") == 0) {+ kvo.bool_value = false;}- std::string arg_next = argv[i];- if (arg_next == "none") {- params.split_mode = LLAMA_SPLIT_MODE_NONE;- } else if (arg_next == "layer") {- params.split_mode = LLAMA_SPLIT_MODE_LAYER;- } else if (arg_next == "row") {- params.split_mode = LLAMA_SPLIT_MODE_ROW;- } else {+ else {+ fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);invalid_param = true;- break;+ return true;}+ }+ else {+ fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);+ invalid_param = true;+ return true;+ }+ params.kv_overrides.push_back(kvo);+ return true;+ }+#ifndef LOG_DISABLE_LOGS+ // Parse args for logging parameters+ if (log_param_single_parse(argv[i])) {+ // Do nothing, log_param_single_parse automatically does it's thing+ // and returns if a match was found and parsed.+ return true;+ }+ if (log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i])) {+ // We have a matching known parameter requiring an argument,+ // now we need to check if there is anything after this argv+ // and flag invalid_param or parse it.+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ if (!log_param_pair_parse( /*check_but_dont_parse*/ false, argv[i - 1], argv[i])) {+ invalid_param = true;+ return true;+ }+ return true;+ }+ // End of Parse args for logging parameters+#endif // LOG_DISABLE_LOGS- } else if (arg == "--tensor-split" || arg == "-ts") {- passed_gpu_flags = true;- if (++i >= argc) {- invalid_param = true;- break;- }- std::string arg_next = argv[i];+ return false;+}- // split string by , and /- const std::regex regex{R"([,/]+)"};- std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1};- std::vector<std::string> split_arg{it, {}};- if (split_arg.size() >= llama_max_devices()) {- invalid_param = true;- break;- }- for (size_t i = 0; i < llama_max_devices(); ++i) {- if (i < split_arg.size()) {- params.tensor_split[i] = std::stof(split_arg[i]);- } else {- params.tensor_split[i] = 0.0f;- }- }- } else if (arg == "--no-mmap") {- params.use_mmap = false;- } else if (arg == "--numa") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::string value(argv[i]);- /**/ if (value == "distribute" || value == "") { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }- else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }- else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }- else { invalid_param = true; break; }- } else if (arg == "--verbose-prompt") {- params.verbose_prompt = true;- } else if (arg == "--no-display-prompt" || arg == "--silent-prompt") {- params.display_prompt = false;- } else if (arg == "-r" || arg == "--reverse-prompt") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.antiprompt.emplace_back(argv[i]);- } else if (arg == "-ld" || arg == "--logdir") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.logdir = argv[i];+bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {+ bool invalid_param = false;+ std::string arg;+ const std::string arg_prefix = "--";+ llama_sampling_params & sparams = params.sparams;- if (params.logdir.back() != DIRECTORY_SEPARATOR) {- params.logdir += DIRECTORY_SEPARATOR;- }- } else if (arg == "--save-all-logits" || arg == "--kl-divergence-base") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.logits_file = argv[i];- } else if (arg == "--perplexity" || arg == "--all-logits") {- params.logits_all = true;- } else if (arg == "--ppl-stride") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.ppl_stride = std::stoi(argv[i]);- } else if (arg == "-ptc" || arg == "--print-token-count") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.n_print = std::stoi(argv[i]);- } else if (arg == "--ppl-output-type") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.ppl_output_type = std::stoi(argv[i]);- } else if (arg == "--hellaswag") {- params.hellaswag = true;- } else if (arg == "--hellaswag-tasks") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.hellaswag_tasks = std::stoi(argv[i]);- } else if (arg == "--winogrande") {- params.winogrande = true;- } else if (arg == "--winogrande-tasks") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.winogrande_tasks = std::stoi(argv[i]);- } else if (arg == "--multiple-choice") {- params.multiple_choice = true;- } else if (arg == "--multiple-choice-tasks") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.multiple_choice_tasks = std::stoi(argv[i]);- } else if (arg == "--kl-divergence") {- params.kl_divergence = true;- } else if (arg == "--ignore-eos") {- params.ignore_eos = true;- } else if (arg == "--no-penalize-nl") {- sparams.penalize_nl = false;- } else if (arg == "-l" || arg == "--logit-bias") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::stringstream ss(argv[i]);- llama_token key;- char sign;- std::string value_str;- try {- if (ss >> key && ss >> sign && std::getline(ss, value_str) && (sign == '+' || sign == '-')) {- sparams.logit_bias[key] = std::stof(value_str) * ((sign == '-') ? -1.0f : 1.0f);- } else {- throw std::exception();- }- } catch (const std::exception&) {- invalid_param = true;- break;- }- } else if (arg == "-h" || arg == "--help") {- return false;+ for (int i = 1; i < argc; i++) {+ arg = argv[i];+ if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {+ std::replace(arg.begin(), arg.end(), '_', '-');+ }- } else if (arg == "--version") {- fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);- fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET);- exit(0);- } else if (arg == "--random-prompt") {- params.random_prompt = true;- } else if (arg == "--in-prefix-bos") {- params.input_prefix_bos = true;- } else if (arg == "--in-prefix") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.input_prefix = argv[i];- } else if (arg == "--in-suffix") {- if (++i >= argc) {- invalid_param = true;- break;- }- params.input_suffix = argv[i];- } else if (arg == "--grammar") {- if (++i >= argc) {- invalid_param = true;- break;- }- sparams.grammar = argv[i];- } else if (arg == "--grammar-file") {- if (++i >= argc) {- invalid_param = true;- break;- }- std::ifstream file(argv[i]);- if (!file) {- fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);- invalid_param = true;- break;- }- std::copy(- std::istreambuf_iterator<char>(file),- std::istreambuf_iterator<char>(),- std::back_inserter(sparams.grammar)- );- } else if (arg == "--override-kv") {- if (++i >= argc) {- invalid_param = true;- break;- }- char * sep = strchr(argv[i], '=');- if (sep == nullptr || sep - argv[i] >= 128) {- fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);- invalid_param = true;- break;- }- struct llama_model_kv_override kvo;- std::strncpy(kvo.key, argv[i], sep - argv[i]);- kvo.key[sep - argv[i]] = 0;- sep++;- if (strncmp(sep, "int:", 4) == 0) {- sep += 4;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;- kvo.int_value = std::atol(sep);- } else if (strncmp(sep, "float:", 6) == 0) {- sep += 6;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;- kvo.float_value = std::atof(sep);- } else if (strncmp(sep, "bool:", 5) == 0) {- sep += 5;- kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;- if (std::strcmp(sep, "true") == 0) {- kvo.bool_value = true;- } else if (std::strcmp(sep, "false") == 0) {- kvo.bool_value = false;- } else {- fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);- invalid_param = true;- break;- }- } else {- fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);- invalid_param = true;- break;- }- params.kv_overrides.push_back(kvo);-#ifndef LOG_DISABLE_LOGS- // Parse args for logging parameters- } else if ( log_param_single_parse( argv[i] ) ) {- // Do nothing, log_param_single_parse automatically does it's thing- // and returns if a match was found and parsed.- } else if ( log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i] ) ) {- // We have a matching known parameter requiring an argument,- // now we need to check if there is anything after this argv- // and flag invalid_param or parse it.- if (++i >= argc) {- invalid_param = true;- break;- }- if( !log_param_pair_parse( /*check_but_dont_parse*/ false, argv[i-1], argv[i]) ) {- invalid_param = true;- break;- }- // End of Parse args for logging parameters-#endif // LOG_DISABLE_LOGS- } else {+ if (!gpt_params_find_arg(argc, argv, arg, params, i, invalid_param)) {throw std::invalid_argument("error: unknown argument: " + arg);}}@@ -983,7 +1295,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" binary file containing multiple choice tasks.\n");printf(" -n N, --n-predict N number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)\n", params.n_predict);printf(" -c N, --ctx-size N size of the prompt context (default: %d, 0 = loaded from model)\n", params.n_ctx);- printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);+ printf(" -b N, --batch-size N logical maximum batch size (default: %d)\n", params.n_batch);+ printf(" -ub N, --ubatch-size N\n");+ printf(" physical maximum batch size (default: %d)\n", params.n_ubatch);printf(" --samplers samplers that will be used for generation in the order, separated by \';\'\n");printf(" (default: %s)\n", sampler_type_names.c_str());printf(" --sampling-seq simplified sequence for samplers that will be used (default: %s)\n", sampler_type_chars.c_str());@@ -1024,12 +1338,14 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n");printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow);printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast);+ printf(" --pooling {none,mean,cls}\n");+ printf(" pooling type for embeddings, use model default if unspecified\n");printf(" -dt N, --defrag-thold N\n");printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold);printf(" --ignore-eos ignore end of stream token and continue generating (implies --logit-bias 2-inf)\n");printf(" --no-penalize-nl do not penalize newline token\n");printf(" --temp N temperature (default: %.1f)\n", (double)sparams.temp);- printf(" --logits-all return logits for all tokens in the batch (default: disabled)\n");+ printf(" --all-logits return logits for all tokens in the batch (default: disabled)\n");printf(" --hellaswag compute HellaSwag score over random tasks from datafile supplied with -f\n");printf(" --hellaswag-tasks N number of tasks to use when computing the HellaSwag score (default: %zu)\n", params.hellaswag_tasks);printf(" --winogrande compute Winogrande score over random tasks from datafile supplied with -f\n");@@ -1042,7 +1358,6 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --chunks N max number of chunks to process (default: %d, -1 = all)\n", params.n_chunks);printf(" -np N, --parallel N number of parallel sequences to decode (default: %d)\n", params.n_parallel);printf(" -ns N, --sequences N number of sequences to decode (default: %d)\n", params.n_sequences);- printf(" -pa N, --p-accept N speculative decoding accept probability (default: %.1f)\n", (double)params.p_accept);printf(" -ps N, --p-split N speculative decoding split probability (default: %.1f)\n", (double)params.p_split);printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA. see examples/llava/README.md\n");@@ -1059,7 +1374,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" - numactl: use the CPU map provided by numactl\n");printf(" if run without this previously, it is recommended to drop the system page cache before using this\n");printf(" see https://github.com/ggerganov/llama.cpp/issues/1437\n");- if (llama_supports_gpu_offload()) {+ // if (llama_supports_gpu_offload()) {printf(" -ngl N, --n-gpu-layers N\n");printf(" number of layers to store in VRAM\n");printf(" -ngld N, --n-gpu-layers-draft N\n");@@ -1073,7 +1388,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu);- }+ // }printf(" --verbose-prompt print a verbose prompt before generation (default: %s)\n", params.verbose_prompt ? "true" : "false");printf(" --no-display-prompt don't print prompt at generation (default: %s)\n", !params.display_prompt ? "true" : "false");printf(" -gan N, --grp-attn-n N\n");@@ -1092,8 +1407,16 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");printf(" --lora-scaled FNAME S apply LoRA adapter with user defined scaling S (implies --no-mmap)\n");printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");+ printf(" --control-vector FNAME\n");+ printf(" add a control vector\n");+ printf(" --control-vector-scaled FNAME S\n");+ printf(" add a control vector with user defined scaling S\n");+ printf(" --control-vector-layer-range START END\n");+ printf(" layer range to apply the control vector(s) to, start and end inclusive\n");printf(" -m FNAME, --model FNAME\n");printf(" model path (default: %s)\n", params.model.c_str());+ printf(" -mu MODEL_URL, --model-url MODEL_URL\n");+ printf(" model download url (default: %s)\n", params.model_url.c_str());printf(" -md FNAME, --model-draft FNAME\n");printf(" draft model for speculative decoding\n");printf(" -ld LOGDIR, --logdir LOGDIR\n");@@ -1278,6 +1601,9 @@ static ggml_type kv_cache_type_from_str(const std::string & s) {if (s == "q4_1") {return GGML_TYPE_Q4_1;}+ if (s == "iq4_nl") {+ return GGML_TYPE_IQ4_NL;+ }if (s == "q5_0") {return GGML_TYPE_Q5_0;}@@ -1292,13 +1618,14 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_paramauto cparams = llama_context_default_params();cparams.n_ctx = params.n_ctx;+ cparams.n_seq_max = params.n_parallel;cparams.n_batch = params.n_batch;+ cparams.n_ubatch = params.n_ubatch;cparams.n_threads = params.n_threads;cparams.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch;- cparams.mul_mat_q = params.mul_mat_q;cparams.seed = params.seed;cparams.logits_all = params.logits_all;- cparams.embedding = params.embedding;+ cparams.embeddings = params.embedding;cparams.rope_scaling_type = params.rope_scaling_type;cparams.rope_freq_base = params.rope_freq_base;cparams.rope_freq_scale = params.rope_freq_scale;@@ -1307,6 +1634,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_paramcparams.yarn_beta_fast = params.yarn_beta_fast;cparams.yarn_beta_slow = params.yarn_beta_slow;cparams.yarn_orig_ctx = params.yarn_orig_ctx;+ cparams.pooling_type = params.pooling_type;cparams.defrag_thold = params.defrag_thold;cparams.offload_kqv = !params.no_kv_offload;@@ -1337,10 +1665,222 @@ void llama_batch_add(batch.n_tokens++;}+#ifdef LLAMA_USE_CURL++struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model,+ struct llama_model_params params) {+ // Basic validation of the model_url+ if (!model_url || strlen(model_url) == 0) {+ fprintf(stderr, "%s: invalid model_url\n", __func__);+ return NULL;+ }++ // Initialize libcurl globally+ auto curl = curl_easy_init();++ if (!curl) {+ fprintf(stderr, "%s: error initializing libcurl\n", __func__);+ return NULL;+ }++ // Set the URL, allow to follow http redirection+ curl_easy_setopt(curl, CURLOPT_URL, model_url);+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);+#if defined(_WIN32)+ // CURLSSLOPT_NATIVE_CA tells libcurl to use standard certificate store of+ // operating system. Currently implemented under MS-Windows.+ curl_easy_setopt(curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA);+#endif++ // Check if the file already exists locally+ struct stat model_file_info;+ auto file_exists = (stat(path_model, &model_file_info) == 0);++ // If the file exists, check for ${path_model}.etag or ${path_model}.lastModified files+ char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};+ char etag_path[LLAMA_CURL_MAX_PATH_LENGTH] = {0};+ snprintf(etag_path, sizeof(etag_path), "%s.etag", path_model);++ char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};+ char last_modified_path[LLAMA_CURL_MAX_PATH_LENGTH] = {0};+ snprintf(last_modified_path, sizeof(last_modified_path), "%s.lastModified", path_model);++ if (file_exists) {+ auto * f_etag = fopen(etag_path, "r");+ if (f_etag) {+ if (!fgets(etag, sizeof(etag), f_etag)) {+ fprintf(stderr, "%s: unable to read file %s\n", __func__, etag_path);+ } else {+ fprintf(stderr, "%s: previous model file found %s: %s\n", __func__, etag_path, etag);+ }+ fclose(f_etag);+ }++ auto * f_last_modified = fopen(last_modified_path, "r");+ if (f_last_modified) {+ if (!fgets(last_modified, sizeof(last_modified), f_last_modified)) {+ fprintf(stderr, "%s: unable to read file %s\n", __func__, last_modified_path);+ } else {+ fprintf(stderr, "%s: previous model file found %s: %s\n", __func__, last_modified_path,+ last_modified);+ }+ fclose(f_last_modified);+ }+ }++ // Send a HEAD request to retrieve the etag and last-modified headers+ struct llama_load_model_from_url_headers {+ char etag[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};+ char last_modified[LLAMA_CURL_MAX_HEADER_LENGTH] = {0};+ };+ llama_load_model_from_url_headers headers;+ {+ typedef size_t(*CURLOPT_HEADERFUNCTION_PTR)(char *, size_t, size_t, void *);+ auto header_callback = [](char * buffer, size_t /*size*/, size_t n_items, void * userdata) -> size_t {+ llama_load_model_from_url_headers *headers = (llama_load_model_from_url_headers *) userdata;++ const char * etag_prefix = "etag: ";+ if (strncmp(buffer, etag_prefix, strlen(etag_prefix)) == 0) {+ strncpy(headers->etag, buffer + strlen(etag_prefix), n_items - strlen(etag_prefix) - 2); // Remove CRLF+ }++ const char * last_modified_prefix = "last-modified: ";+ if (strncmp(buffer, last_modified_prefix, strlen(last_modified_prefix)) == 0) {+ strncpy(headers->last_modified, buffer + strlen(last_modified_prefix),+ n_items - strlen(last_modified_prefix) - 2); // Remove CRLF+ }+ return n_items;+ };++ curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); // will trigger the HEAD verb+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); // hide head request progress+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, static_cast<CURLOPT_HEADERFUNCTION_PTR>(header_callback));+ curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);++ CURLcode res = curl_easy_perform(curl);+ if (res != CURLE_OK) {+ curl_easy_cleanup(curl);+ fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));+ return NULL;+ }++ long http_code = 0;+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);+ if (http_code != 200) {+ // HEAD not supported, we don't know if the file has changed+ // force trigger downloading+ file_exists = false;+ fprintf(stderr, "%s: HEAD invalid http status code received: %ld\n", __func__, http_code);+ }+ }++ // If the ETag or the Last-Modified headers are different: trigger a new download+ if (!file_exists || strcmp(etag, headers.etag) != 0 || strcmp(last_modified, headers.last_modified) != 0) {+ char path_model_temporary[LLAMA_CURL_MAX_PATH_LENGTH] = {0};+ snprintf(path_model_temporary, sizeof(path_model_temporary), "%s.downloadInProgress", path_model);+ if (file_exists) {+ fprintf(stderr, "%s: deleting previous downloaded model file: %s\n", __func__, path_model);+ if (remove(path_model) != 0) {+ curl_easy_cleanup(curl);+ fprintf(stderr, "%s: unable to delete file: %s\n", __func__, path_model);+ return NULL;+ }+ }++ // Set the output file+ auto * outfile = fopen(path_model_temporary, "wb");+ if (!outfile) {+ curl_easy_cleanup(curl);+ fprintf(stderr, "%s: error opening local file for writing: %s\n", __func__, path_model);+ return NULL;+ }++ typedef size_t(*CURLOPT_WRITEFUNCTION_PTR)(void * data, size_t size, size_t nmemb, void * fd);+ auto write_callback = [](void * data, size_t size, size_t nmemb, void * fd) -> size_t {+ return fwrite(data, size, nmemb, (FILE *)fd);+ };+ curl_easy_setopt(curl, CURLOPT_NOBODY, 0L);+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<CURLOPT_WRITEFUNCTION_PTR>(write_callback));+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile);++ // display download progress+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);++ // start the download+ fprintf(stderr, "%s: downloading model from %s to %s (server_etag:%s, server_last_modified:%s)...\n", __func__,+ model_url, path_model, headers.etag, headers.last_modified);+ auto res = curl_easy_perform(curl);+ if (res != CURLE_OK) {+ fclose(outfile);+ curl_easy_cleanup(curl);+ fprintf(stderr, "%s: curl_easy_perform() failed: %s\n", __func__, curl_easy_strerror(res));+ return NULL;+ }++ long http_code = 0;+ curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);+ if (http_code < 200 || http_code >= 400) {+ fclose(outfile);+ curl_easy_cleanup(curl);+ fprintf(stderr, "%s: invalid http status code received: %ld\n", __func__, http_code);+ return NULL;+ }++ // Clean up+ fclose(outfile);++ // Write the new ETag to the .etag file+ if (strlen(headers.etag) > 0) {+ auto * etag_file = fopen(etag_path, "w");+ if (etag_file) {+ fputs(headers.etag, etag_file);+ fclose(etag_file);+ fprintf(stderr, "%s: model etag saved %s: %s\n", __func__, etag_path, headers.etag);+ }+ }++ // Write the new lastModified to the .etag file+ if (strlen(headers.last_modified) > 0) {+ auto * last_modified_file = fopen(last_modified_path, "w");+ if (last_modified_file) {+ fputs(headers.last_modified, last_modified_file);+ fclose(last_modified_file);+ fprintf(stderr, "%s: model last modified saved %s: %s\n", __func__, last_modified_path,+ headers.last_modified);+ }+ }++ if (rename(path_model_temporary, path_model) != 0) {+ curl_easy_cleanup(curl);+ fprintf(stderr, "%s: unable to rename file: %s to %s\n", __func__, path_model_temporary, path_model);+ return NULL;+ }+ }++ curl_easy_cleanup(curl);++ return llama_load_model_from_file(path_model, params);+}++#else++struct llama_model * llama_load_model_from_url(const char * /*model_url*/, const char * /*path_model*/,+ struct llama_model_params /*params*/) {+ fprintf(stderr, "%s: llama.cpp built without libcurl, downloading from an url not supported.\n", __func__);+ return nullptr;+}++#endif // LLAMA_USE_CURL+std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_params(gpt_params & params) {auto mparams = llama_model_params_from_gpt_params(params);- llama_model * model = llama_load_model_from_file(params.model.c_str(), mparams);+ llama_model * model = nullptr;+ if (!params.model_url.empty()) {+ model = llama_load_model_from_url(params.model_url.c_str(), params.model.c_str(), mparams);+ } else {+ model = llama_load_model_from_file(params.model.c_str(), mparams);+ }if (model == NULL) {fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str());return std::make_tuple(nullptr, nullptr);@@ -1355,6 +1895,30 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_parreturn std::make_tuple(nullptr, nullptr);}+ if (!params.control_vectors.empty()) {+ if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;+ if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_n_layer(model);++ const auto cvec = llama_control_vector_load(params.control_vectors);+ if (cvec.n_embd == -1) {+ llama_free(lctx);+ llama_free_model(model);+ return std::make_tuple(nullptr, nullptr);+ }++ int err = llama_control_vector_apply(lctx,+ cvec.data.data(),+ cvec.data.size(),+ cvec.n_embd,+ params.control_vector_layer_start,+ params.control_vector_layer_end);+ if (err) {+ llama_free(lctx);+ llama_free_model(model);+ return std::make_tuple(nullptr, nullptr);+ }+ }+for (unsigned int i = 0; i < params.lora_adapter.size(); ++i) {const std::string& lora_adapter = std::get<0>(params.lora_adapter[i]);float lora_scale = std::get<1>(params.lora_adapter[i]);@@ -1383,6 +1947,7 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_parstd::vector<llama_token> tmp = { llama_token_bos(model), llama_token_eos(model), };llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0));llama_kv_cache_clear(lctx);+ llama_synchronize(lctx);llama_reset_timings(lctx);}@@ -1739,7 +2304,6 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "n_predict: %d # default: -1 (unlimited)\n", params.n_predict);fprintf(stream, "n_probs: %d # only used by server binary, default: 0\n", sparams.n_probs);fprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false");- fprintf(stream, "no_mul_mat_q: %s # default: false\n", !params.mul_mat_q ? "true" : "false");fprintf(stream, "no_penalize_nl: %s # default: false\n", !sparams.penalize_nl ? "true" : "false");fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type);fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride);@@ -1791,17 +2355,17 @@ void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size) {static const char slot_chars[] = ".123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+";printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d",- view.n_cells, view.n_max_seq, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);+ view.n_cells, view.n_seq_max, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);llama_kv_cache_view_cell * c_curr = view.cells;llama_seq_id * cs_curr = view.cells_sequences;- for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) {+ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) {if (i % row_size == 0) {printf("\n%5d: ", i);}int seq_count = 0;- for (int j = 0; j < view.n_max_seq; j++) {+ for (int j = 0; j < view.n_seq_max; j++) {if (cs_curr[j] >= 0) { seq_count++; }}putchar(slot_chars[std::min(sizeof(slot_chars) - 2, size_t(seq_count))]);@@ -1814,14 +2378,14 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) {static const char slot_chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d\n",- view.n_cells, view.n_max_seq, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);+ view.n_cells, view.n_seq_max, view.used_cells, view.token_count, view.max_contiguous, view.max_contiguous_idx);std::unordered_map<llama_seq_id, size_t> seqs;llama_kv_cache_view_cell * c_curr = view.cells;llama_seq_id * cs_curr = view.cells_sequences;- for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) {- for (int j = 0; j < view.n_max_seq; j++) {+ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) {+ for (int j = 0; j < view.n_seq_max; j++) {if (cs_curr[j] < 0) { continue; }if (seqs.find(cs_curr[j]) == seqs.end()) {if (seqs.size() + 1 >= sizeof(slot_chars)) { break; }@@ -1840,11 +2404,11 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) {c_curr = view.cells;cs_curr = view.cells_sequences;- for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_max_seq) {+ for (int i = 0; i < view.n_cells; i++, c_curr++, cs_curr += view.n_seq_max) {if (i % row_size == 0) {printf("\n%5d: ", i);}- for (int j = 0; j < view.n_max_seq; j++) {+ for (int j = 0; j < view.n_seq_max; j++) {if (cs_curr[j] >= 0) {const auto & it = seqs.find(cs_curr[j]);putchar(it != seqs.end() ? int(slot_chars[it->second]) : '+');@@ -1857,3 +2421,207 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) {printf("\n=== Done dumping\n");}++void llama_embd_normalize(const float * inp, float * out, int n) {+ double sum = 0.0;+ for (int i = 0; i < n; i++) {+ sum += inp[i] * inp[i];+ }+ sum = sqrt(sum);++ const float norm = sum > 0.0 ? 1.0f / sum : 0.0f;++ for (int i = 0; i < n; i++) {+ out[i] = inp[i] * norm;+ }+}++float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n){+ double sum = 0.0;+ double sum1 = 0.0;+ double sum2 = 0.0;++ for (int i = 0; i < n; i++) {+ sum += embd1[i] * embd2[i];+ sum1 += embd1[i] * embd1[i];+ sum2 += embd2[i] * embd2[i];+ }++ return sum / (sqrt(sum1) * sqrt(sum2));+}++//+// Control vector utils+//++static llama_control_vector_data llama_control_vector_load_one(const llama_control_vector_load_info & load_info) {+ int32_t n_tensors;++ size_t n_bytes = 0;++ uint32_t max_direction_layer = 0;++ llama_control_vector_data result = { -1, {} };++ // calculate size of ctx needed for tensors, ensure tensors are f32, and find max layer+ {+ struct ggml_init_params meta_params = {+ /* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead(),+ /* .mem_buffer = */ nullptr,+ /* .no_alloc = */ true,+ };+ ggml_context * meta_ctx = ggml_init(meta_params);+ struct gguf_init_params meta_gguf_params = {+ /* .no_alloc = */ true,+ /* .ctx = */ &meta_ctx,+ };+ struct llamafile * file = llamafile_open_gguf(load_info.fname.c_str(), "rb");+ if (!file) {+ perror(load_info.fname.c_str());+ ggml_free(meta_ctx);+ return result;+ }+ struct gguf_context * meta_ctx_gguf = gguf_init_from_file(file, meta_gguf_params);+ if (!meta_ctx_gguf) {+ fprintf(stderr, "%s: failed to load control vector from %s\n", __func__, load_info.fname.c_str());+ llamafile_close(file);+ ggml_free(meta_ctx);+ return result;+ }++ n_tensors = gguf_get_n_tensors(meta_ctx_gguf);+ for (int i = 0; i < n_tensors; i++) {+ std::string name = gguf_get_tensor_name(meta_ctx_gguf, i);++ // split on '.'+ size_t dotpos = name.find('.');+ if (dotpos != std::string::npos && name.substr(0, dotpos) == "direction") {+ try {+ uint32_t layer = std::stoi(name.substr(dotpos + 1));+ if (layer == 0) {+ fprintf(stderr, "%s: direction tensor invalid in %s\n", __func__, load_info.fname.c_str());+ ggml_free(meta_ctx);+ gguf_free(meta_ctx_gguf);+ llamafile_close(file);+ return result;+ }+ if (layer > max_direction_layer) {+ max_direction_layer = layer;+ }+ } catch (...) {+ fprintf(stderr, "%s: direction tensor invalid in %s\n", __func__, load_info.fname.c_str());+ ggml_free(meta_ctx);+ gguf_free(meta_ctx_gguf);+ llamafile_close(file);+ return result;+ }+ }++ struct ggml_tensor * tensor_meta = ggml_get_tensor(meta_ctx, name.c_str());+ if (tensor_meta->type != GGML_TYPE_F32 || ggml_n_dims(tensor_meta) != 1) {+ fprintf(stderr, "%s: direction tensor invalid in %s\n", __func__, load_info.fname.c_str());+ ggml_free(meta_ctx);+ gguf_free(meta_ctx_gguf);+ llamafile_close(file);+ return result;+ }+ if (result.n_embd == -1) {+ result.n_embd = ggml_nelements(tensor_meta);+ } else if (ggml_nelements(tensor_meta) != result.n_embd) {+ fprintf(stderr, "%s: direction tensor sizes mismatched in %s\n", __func__, load_info.fname.c_str());+ ggml_free(meta_ctx);+ gguf_free(meta_ctx_gguf);+ llamafile_close(file);+ return result;+ }+ n_bytes += ggml_nbytes(tensor_meta);+ }+ ggml_free(meta_ctx);+ gguf_free(meta_ctx_gguf);+ llamafile_close(file);+ }++ if (n_tensors == 0) {+ fprintf(stderr, "%s: no direction tensors found in %s\n", __func__, load_info.fname.c_str());+ return result;+ }++ // load and scale tensors into final control vector context+ struct ggml_init_params ggml_params = {+ /* .mem_size = */ ggml_tensor_overhead() * n_tensors + n_bytes,+ /* .mem_buffer = */ nullptr,+ /* .no_alloc = */ false,+ };+ struct ggml_context * ctx = ggml_init(ggml_params);++ struct gguf_init_params params = {+ /*.no_alloc = */ false,+ /*.ctx = */ &ctx,+ };+ struct llamafile * file = llamafile_open_gguf(load_info.fname.c_str(), "rb");+ if (!file) {+ perror(load_info.fname.c_str());+ ggml_free(ctx);+ return result;+ }+ struct gguf_context * ctx_gguf = gguf_init_from_file(file, params);+ if (!ctx_gguf) {+ fprintf(stderr, "%s: failed to load control vector from %s\n", __func__, load_info.fname.c_str());+ ggml_free(ctx);+ llamafile_close(file);+ return result;+ }++ // do not store data for layer 0 (it's not used)+ result.data.resize(result.n_embd * max_direction_layer);++ for (uint32_t il = 1; il <= max_direction_layer; il++) {+ const std::string name = "direction." + std::to_string(il);+ const ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());++ float * dst = result.data.data() + result.n_embd * (il - 1);++ if (tensor) {+ const float * src = (const float *) tensor->data;+ for (int j = 0; j < result.n_embd; j++) {+ dst[j] = src[j] * load_info.strength;+ }+ } else {+ for (int j = 0; j < result.n_embd; j++) {+ dst[j] = 0.0f;+ }+ }+ }++ return result;+}++llama_control_vector_data llama_control_vector_load(const std::vector<llama_control_vector_load_info> & load_infos) {+ llama_control_vector_data result = { -1, {} };++ for (const auto & info : load_infos) {+ auto cur = llama_control_vector_load_one(info);++ if (cur.n_embd == -1) {+ return result;+ }+ if (result.n_embd != -1 && (result.n_embd != cur.n_embd || result.data.size() != cur.data.size())) {+ fprintf(stderr, "%s: control vector in %s does not match previous vector dimensions\n", __func__, info.fname.c_str());+ return result;+ }++ if (result.n_embd == -1) {+ result = std::move(cur);+ } else {+ for (size_t i = 0; i < cur.data.size(); i++) {+ result.data[i] += cur.data[i];+ }+ }+ }++ if (result.n_embd == -1) {+ fprintf(stderr, "%s: no vectors passed\n", __func__);+ }++ return result;+}diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex c3eb91a5..c9902dc5 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -40,13 +40,16 @@ extern char const *LLAMA_COMMIT;extern char const *LLAMA_COMPILER;extern char const *LLAMA_BUILD_TARGET;+struct llama_control_vector_load_info;++int32_t get_num_physical_cores();+//// CLI argument parsing//-int32_t get_num_physical_cores();struct gpt_params {- uint32_t seed = -1; // RNG seed+ uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seedint32_t n_threads = get_num_physical_cores();int32_t n_threads_draft = -1;@@ -54,13 +57,13 @@ struct gpt_params {int32_t n_threads_batch_draft = -1;int32_t n_predict = -1; // new tokens to predictint32_t n_ctx = 512; // context size- int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)+ int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS)+ int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS)int32_t n_keep = 0; // number of tokens to keep from initial prompt- int32_t n_draft = 8; // number of tokens to draft during speculative decoding+ int32_t n_draft = 5; // number of tokens to draft during speculative decodingint32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)int32_t n_parallel = 1; // number of parallel sequences to decodeint32_t n_sequences = 1; // number of sequences to decode- float p_accept = 0.5f; // speculative decoding accept probabilityfloat p_split = 0.1f; // speculative decoding split probabilityint32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default)int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default)@@ -79,13 +82,17 @@ struct gpt_params {float yarn_beta_slow = 1.0f; // YaRN high correction dimint32_t yarn_orig_ctx = 0; // YaRN original context lengthfloat defrag_thold = -1.0f; // KV cache defragmentation threshold- int32_t rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;- ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED;++ ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED;++ llama_rope_scaling_type rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;+ llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; // pooling type for embeddings// // sampling parametersstruct llama_sampling_params sparams;std::string model = "models/7B/ggml-model-f16.gguf"; // model path+ std::string model_url = ""; // model url to downloadstd::string model_draft = ""; // draft model for speculative decodingstd::string model_alias = "unknown"; // model aliasstd::string prompt = "";@@ -103,6 +110,11 @@ struct gpt_params {std::vector<std::tuple<std::string, float>> lora_adapter; // lora adapter path with user defined scalestd::string lora_base = ""; // base model path for the lora adapter+ std::vector<llama_control_vector_load_info> control_vectors; // control vector with user defined scale++ int32_t control_vector_layer_start = -1; // layer range for control vector+ int32_t control_vector_layer_end = -1; // layer range for control vector+int ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used.int ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line// (which is more convenient to use for plotting)@@ -118,7 +130,6 @@ struct gpt_params {bool kl_divergence = false; // compute KL-divergence- bool mul_mat_q = true; // if true, use mul_mat_q kernels instead of cuBLASbool random_prompt = false; // do not randomize prompt if none providedbool use_color = false; // use color to distinguish generations and inputsbool interactive = false; // interactive mode@@ -184,6 +195,9 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_parstruct llama_model_params llama_model_params_from_gpt_params (const gpt_params & params);struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params);+struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model,+ struct llama_model_params params);+// Batch utilsvoid llama_batch_clear(struct llama_batch & batch);@@ -262,3 +276,32 @@ void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size = 80);// Dump the KV cache view showing individual sequences in each cell (long output).void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40);++//+// Embedding utils+//++void llama_embd_normalize(const float * inp, float * out, int n);++float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n);++//+// Control vector utils+//++struct llama_control_vector_data {+ int n_embd;++ // stores data for layers [1, n_layer] where n_layer = data.size() / n_embd+ std::vector<float> data;+};++struct llama_control_vector_load_info {+ float strength;++ std::string fname;+};++// Load control vectors, scale each by strength, and add them together.+// On error, returns {-1, empty}+llama_control_vector_data llama_control_vector_load(const std::vector<llama_control_vector_load_info> & load_infos);diff --git a/llama.cpp/ggml-alloc.c b/llama.cpp/ggml-alloc.cindex cbffba8a..c9d47bb0 100644--- a/llama.cpp/ggml-alloc.c+++ b/llama.cpp/ggml-alloc.c@@ -67,7 +67,6 @@ static bool ggml_op_can_inplace(enum ggml_op op) {}}-// TODO: GGML_PAD ?static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {assert(alignment && !(alignment & (alignment - 1))); // power of 2size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment;@@ -75,25 +74,14 @@ static size_t aligned_offset(const void * buffer, size_t offset, size_t alignmen}// tallocr-struct ggml_tallocr {- ggml_backend_buffer_t buffer;- void * base;- size_t alignment;- size_t offset;-};--ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer) {- ggml_tallocr_t talloc = malloc(sizeof(struct ggml_tallocr));- if (talloc == NULL) {- return NULL;- }+struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) {void * base = ggml_backend_buffer_get_base(buffer);size_t align = ggml_backend_buffer_get_alignment(buffer);assert(align && !(align & (align - 1))); // power of 2- *talloc = (struct ggml_tallocr) {+ struct ggml_tallocr talloc = (struct ggml_tallocr) {/*.buffer = */ buffer,/*.base = */ base,/*.alignment = */ align,@@ -102,11 +90,7 @@ ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer) {return talloc;}-void ggml_tallocr_free(ggml_tallocr_t talloc) {- free(talloc);-}--void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor) {+void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) {size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor);size = GGML_PAD(size, talloc->alignment);@@ -360,12 +344,16 @@ struct hash_node {bool allocated;};-//struct tensor_alloc {size_t offset;size_t size_max; // 0 = pre-allocated, unused, or view};+struct leaf_alloc {+ int buffer_id;+ struct tensor_alloc leaf;+};+struct node_alloc {int buffer_id;struct tensor_alloc dst;@@ -384,7 +372,7 @@ struct ggml_gallocr {struct node_alloc * node_allocs; // [n_nodes]int n_nodes;- struct tensor_alloc * leaf_allocs; // [n_leafs]+ struct leaf_alloc * leaf_allocs; // [n_leafs]int n_leafs;};@@ -549,17 +537,28 @@ static int get_node_buffer_id(const int * node_buffer_ids, int i) {return node_buffer_ids ? node_buffer_ids[i] : 0;}-static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) {+static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {// clear hash tablesmemset(galloc->hash_set.keys, 0, galloc->hash_set.size * sizeof(struct ggml_tensor *));memset(galloc->hash_values, 0, galloc->hash_set.size * sizeof(struct hash_node));+ // allocate leafs+ // these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes+ for (int i = 0; i < graph->n_leafs; i++) {+ struct ggml_tensor * leaf = graph->leafs[i];+ ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i));+ }+// count number of children and views- // allocate all graph inputs and leafs first to avoid overwriting them+ // allocate other graph inputs and leafs first to avoid overwriting themfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- if (ggml_is_view(node)) {+ // TODO: better way to add external dependencies+ // GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to+ // control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node+ // itself is never used and should not be considered a dependency+ if (ggml_is_view(node) && node->op != GGML_OP_NONE) {struct ggml_tensor * view_src = node->view_src;ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;}@@ -576,26 +575,13 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgrggml_gallocr_hash_get(galloc, src)->n_children += 1;- // allocate explicit inputs and leafs- if (src->flags & GGML_TENSOR_FLAG_INPUT || src->op == GGML_OP_NONE) {+ // allocate explicit inputs+ if (src->flags & GGML_TENSOR_FLAG_INPUT) {ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i));}}}- // allocate the remaining leafs that are unused on the graph- // these are effectively static tensors that the application is not using in the graph, but may still want to allocate for other purposes- for (int i = 0; i < graph->n_leafs; i++) {- struct ggml_tensor * leaf = graph->leafs[i];- struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);-- if (hn->n_children == 0) {- assert(!hn->allocated);- // since buffer ids are only given for nodes, these leafs are always allocated in the first buffer- ggml_gallocr_allocate_node(galloc, leaf, 0);- }- }-// allocate tensorsfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];@@ -658,7 +644,7 @@ static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgr}}-bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids) {+bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {size_t hash_size = graph->visited_hash_table.size;// initialize hash table@@ -682,7 +668,7 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c}// allocate in hash table- ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids);+ ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids);// set the node_allocs from the hash tableif (galloc->n_nodes < graph->n_nodes) {@@ -717,15 +703,16 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c}if (galloc->n_leafs < graph->n_leafs) {free(galloc->leaf_allocs);- galloc->leaf_allocs = calloc(sizeof(struct tensor_alloc), graph->n_leafs);+ galloc->leaf_allocs = calloc(sizeof(galloc->leaf_allocs[0]), graph->n_leafs);GGML_ASSERT(galloc->leaf_allocs != NULL);}galloc->n_leafs = graph->n_leafs;for (int i = 0; i < graph->n_leafs; i++) {struct ggml_tensor * leaf = graph->leafs[i];struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);- galloc->leaf_allocs[i].offset = hn->offset;- galloc->leaf_allocs[i].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);+ galloc->leaf_allocs[i].buffer_id = hn->buffer_id;+ galloc->leaf_allocs[i].leaf.offset = hn->offset;+ galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);}// reallocate buffers if needed@@ -733,7 +720,8 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, csize_t cur_size = galloc->buffers[i] ? ggml_backend_buffer_get_size(galloc->buffers[i]) : 0;size_t new_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i]);- if (new_size > cur_size) {+ // even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views+ if (new_size > cur_size || galloc->buffers[i] == NULL) {#ifndef NDEBUGfprintf(stderr, "%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);#endif@@ -750,30 +738,30 @@ bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, c}bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) {- return ggml_gallocr_reserve_n(galloc, graph, NULL);+ return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL);}-static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, struct tensor_alloc * tensor_alloc) {- assert(node->data || node->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max);+static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, int buffer_id, struct tensor_alloc * tensor_alloc) {+ assert(tensor->data || tensor->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);- if (node->view_src != NULL) {- if (node->buffer == NULL) {+ if (tensor->view_src != NULL) {+ if (tensor->buffer == NULL) {assert(tensor_alloc->offset == SIZE_MAX);- if (node->view_src->buffer == NULL) {+ if (tensor->view_src->buffer == NULL) {// this tensor was allocated without ggml-backendreturn;}- ggml_backend_view_init(galloc->buffers[buffer_id], node);+ ggml_backend_view_init(galloc->buffers[buffer_id], tensor);}} else {- if (node->data == NULL) {+ if (tensor->data == NULL) {assert(tensor_alloc->offset != SIZE_MAX);- assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], node) <= tensor_alloc->size_max);+ assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);void * base = ggml_backend_buffer_get_base(galloc->buffers[buffer_id]);void * addr = (char *)base + tensor_alloc->offset;- ggml_backend_tensor_alloc(galloc->buffers[buffer_id], node, addr);+ ggml_backend_tensor_alloc(galloc->buffers[buffer_id], tensor, addr);} else {- if (node->buffer == NULL) {+ if (tensor->buffer == NULL) {// this tensor was allocated without ggml-backendreturn;}@@ -849,13 +837,18 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph)// reset buffersfor (int i = 0; i < galloc->n_buffers; i++) {- // zero size buffers are not allocatedif (galloc->buffers[i] != NULL) {ggml_backend_buffer_reset(galloc->buffers[i]);}}// allocate the graph tensors from the previous assignments+ // leafs+ for (int i = 0; i < graph->n_leafs; i++) {+ struct ggml_tensor * leaf = graph->leafs[i];+ struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i];+ ggml_gallocr_init_tensor(galloc, leaf, leaf_alloc->buffer_id, &leaf_alloc->leaf);+ }// nodesfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];@@ -869,12 +862,6 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph)}ggml_gallocr_init_tensor(galloc, node, node_alloc->buffer_id, &node_alloc->dst);}- // leafs- for (int i = 0; i < graph->n_leafs; i++) {- struct ggml_tensor * leaf = graph->leafs[i];- struct tensor_alloc * leaf_alloc = &galloc->leaf_allocs[i];- ggml_gallocr_init_tensor(galloc, leaf, 0, leaf_alloc);- }return true;}@@ -906,12 +893,12 @@ static bool alloc_tensor_range(struct ggml_context * ctx,return false;}- struct ggml_tallocr * tallocr = ggml_tallocr_new(buffer);+ struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {if (t->data == NULL) {if (t->view_src == NULL) {- ggml_tallocr_alloc(tallocr, t);+ ggml_tallocr_alloc(&tallocr, t);} else if (t->buffer == NULL) {ggml_backend_view_init(buffer, t);}@@ -923,8 +910,6 @@ static bool alloc_tensor_range(struct ggml_context * ctx,}}- ggml_tallocr_free(tallocr);-*buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1));(*buffers)[(*n_buffers)++] = buffer;diff --git a/llama.cpp/ggml-alloc.h b/llama.cpp/ggml-alloc.hindex 1d9085d1..434c13b3 100644--- a/llama.cpp/ggml-alloc.h+++ b/llama.cpp/ggml-alloc.h@@ -11,11 +11,15 @@ typedef struct ggml_backend_buffer * ggml_backend_buffer_t;typedef struct ggml_backend * ggml_backend_t;// Tensor allocator-typedef struct ggml_tallocr * ggml_tallocr_t;+struct ggml_tallocr {+ ggml_backend_buffer_t buffer;+ void * base;+ size_t alignment;+ size_t offset;+};-GGML_API ggml_tallocr_t ggml_tallocr_new(ggml_backend_buffer_t buffer);-GGML_API void ggml_tallocr_free(ggml_tallocr_t talloc);-GGML_API void ggml_tallocr_alloc(ggml_tallocr_t talloc, struct ggml_tensor * tensor);+GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);+GGML_API void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);// Graph allocator/*@@ -50,7 +54,11 @@ GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed// returns false if the buffer allocation failedGGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);-GGML_API bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids);+GGML_API bool ggml_gallocr_reserve_n(+ ggml_gallocr_t galloc,+ struct ggml_cgraph * graph,+ const int * node_buffer_ids,+ const int * leaf_buffer_ids);// automatic reallocation if the topology changes when using a single buffer// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)diff --git a/llama.cpp/ggml-backend-impl.h b/llama.cpp/ggml-backend-impl.hindex 0e5b6850..a1f1741e 100644--- a/llama.cpp/ggml-backend-impl.h+++ b/llama.cpp/ggml-backend-impl.h@@ -88,29 +88,48 @@ extern "C" {// (optional) asynchronous tensor data accessvoid (*GGML_CALL set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);- bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst);+ bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);// (optional) complete all pending operationsvoid (*GGML_CALL synchronize)(ggml_backend_t backend);- // compute graph with a plan+ // compute graph with a plan (not used currently)ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);- void (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);+ // compute graph with a plan+ enum ggml_status (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);// compute graph without a plan (async)- bool (*GGML_CALL graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph);+ enum ggml_status (*GGML_CALL graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph);// check if the backend supports an operationbool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);++ // check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer+ // these should be expensive operations with large batch sizes that may benefit from running on this backend+ // even if the weight has to be copied from the CPU temporarily+ bool (*GGML_CALL offload_op)(ggml_backend_t backend, const struct ggml_tensor * op);++ // (optional) event synchronization+ ggml_backend_event_t (*GGML_CALL event_new) (ggml_backend_t backend);+ void (*GGML_CALL event_free) (ggml_backend_event_t event);+ void (*GGML_CALL event_record) (ggml_backend_event_t event);+ void (*GGML_CALL event_wait) (ggml_backend_t backend, ggml_backend_event_t event);+ void (*GGML_CALL event_synchronize) (ggml_backend_event_t event);};struct ggml_backend {- struct ggml_backend_i iface;+ ggml_guid_t guid;+ struct ggml_backend_i iface;ggml_backend_context_t context;};+ struct ggml_backend_event {+ ggml_backend_t backend;+ void * context;+ };+//// Backend registry//diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex e21683ad..2954aa76 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -171,6 +171,13 @@ bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml// backend+ggml_guid_t ggml_backend_guid(ggml_backend_t backend) {+ if (backend == NULL) {+ return NULL;+ }+ return backend->guid;+}+const char * ggml_backend_name(ggml_backend_t backend) {if (backend == NULL) {return "NULL";@@ -227,29 +234,29 @@ void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tenGGML_CALL void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(buf != NULL && "tensor buffer not set");+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");if (!size) {return;}- tensor->buffer->iface.set_tensor(buf, tensor, data, offset, size);+ buf->iface.set_tensor(buf, tensor, data, offset, size);}GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;+ GGML_ASSERT(buf != NULL && "tensor buffer not set");GGML_ASSERT(tensor->data != NULL && "tensor not allocated");- GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");if (!size) {return;}- tensor->buffer->iface.get_tensor(buf, tensor, data, offset, size);+ buf->iface.get_tensor(buf, tensor, data, offset, size);}void ggml_backend_synchronize(ggml_backend_t backend) {@@ -261,18 +268,30 @@ void ggml_backend_synchronize(ggml_backend_t backend) {}ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+ GGML_ASSERT(backend->iface.graph_plan_create != NULL);+return backend->iface.graph_plan_create(backend, cgraph);}void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {+ GGML_ASSERT(backend->iface.graph_plan_free != NULL);+backend->iface.graph_plan_free(backend, plan);}-void ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {- backend->iface.graph_plan_compute(backend, plan);+enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {+ GGML_ASSERT(backend->iface.graph_plan_compute != NULL);++ return backend->iface.graph_plan_compute(backend, plan);+}++enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+ enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);+ ggml_backend_synchronize(backend);+ return err;}-bool ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) {return backend->iface.graph_compute(backend, cgraph);}@@ -280,6 +299,13 @@ bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor *return backend->iface.supports_op(backend, op);}+bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) {+ if (backend->iface.offload_op != NULL) {+ return backend->iface.offload_op(backend, op);+ }+ return false;+}+// backend copystatic bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {@@ -320,34 +346,68 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst}}-void ggml_backend_tensor_copy_async(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) {+void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst) {GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");if (src == dst) {return;}- if (ggml_backend_buft_supports_backend(src->buffer->buft, backend) && ggml_backend_buft_supports_backend(dst->buffer->buft, backend)) {- if (backend->iface.cpy_tensor_async != NULL) {- if (backend->iface.cpy_tensor_async(backend, src, dst)) {- return;- }+ if (backend_dst->iface.cpy_tensor_async != NULL) {+ if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {+ return;}}- size_t nbytes = ggml_nbytes(src);+ // an async copy would normally happen after all the queued operations on both backends are completed+ // sync src, set_async dstif (ggml_backend_buffer_is_host(src->buffer)) {- ggml_backend_tensor_set_async(backend, dst, src->data, 0, nbytes);- }- else {+ ggml_backend_synchronize(backend_src);+ ggml_backend_tensor_set_async(backend_dst, dst, src->data, 0, ggml_nbytes(src));+ } else {+ ggml_backend_synchronize(backend_src);ggml_backend_tensor_copy(src, dst);+ ggml_backend_synchronize(backend_dst);+ }+}++// events++ggml_backend_event_t ggml_backend_event_new(ggml_backend_t backend) {+ if (backend->iface.event_new == NULL) {+ return NULL;+ }+ return backend->iface.event_new(backend);+}++void ggml_backend_event_free(ggml_backend_event_t event) {+ if (event == NULL) {+ return;}+ event->backend->iface.event_free(event);+}++void ggml_backend_event_record(ggml_backend_event_t event) {+ GGML_ASSERT(event->backend->iface.event_record != NULL);++ event->backend->iface.event_record(event);+}++void ggml_backend_event_synchronize(ggml_backend_event_t event) {+ GGML_ASSERT(event->backend->iface.event_synchronize != NULL);++ event->backend->iface.event_synchronize(event);}+void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {+ GGML_ASSERT(backend->iface.event_wait != NULL);++ backend->iface.event_wait(backend, event);+}// backend registry-#define GGML_MAX_BACKENDS_REG 16+#define GGML_REG_MAX_BACKENDS 16struct ggml_backend_reg {char name[128];@@ -356,7 +416,7 @@ struct ggml_backend_reg {void * user_data;};-static struct ggml_backend_reg ggml_backend_registry[GGML_MAX_BACKENDS_REG];+static struct ggml_backend_reg ggml_backend_registry[GGML_REG_MAX_BACKENDS];static size_t ggml_backend_registry_count = 0;GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data);@@ -401,7 +461,7 @@ GGML_CALL static void ggml_backend_registry_init(void) {}GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {- GGML_ASSERT(ggml_backend_registry_count < GGML_MAX_BACKENDS_REG);+ GGML_ASSERT(ggml_backend_registry_count < GGML_REG_MAX_BACKENDS);size_t id = ggml_backend_registry_count;@@ -721,6 +781,10 @@ GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggif (cpu_plan->cplan.work_size > 0) {cpu_plan->cplan.work_data = malloc(cpu_plan->cplan.work_size);+ if (cpu_plan->cplan.work_data == NULL) {+ free(cpu_plan);+ return NULL;+ }}cpu_plan->cplan.abort_callback = cpu_ctx->abort_callback;@@ -738,22 +802,26 @@ GGML_CALL static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, gGGML_UNUSED(backend);}-GGML_CALL static void ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {+GGML_CALL static enum ggml_status ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;- ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);+ return ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);GGML_UNUSED(backend);}-GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+GGML_CALL static enum ggml_status ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads);if (cpu_ctx->work_size < cplan.work_size) {- // TODO: may be faster to free and use malloc to avoid the copy- cpu_ctx->work_data = realloc(cpu_ctx->work_data, cplan.work_size);+ free(cpu_ctx->work_data);+ cpu_ctx->work_data = malloc(cplan.work_size);+ if (cpu_ctx->work_data == NULL) {+ cpu_ctx->work_size = 0;+ return GGML_STATUS_ALLOC_FAILED;+ }cpu_ctx->work_size = cplan.work_size;}cplan.work_data = cpu_ctx->work_data;@@ -761,8 +829,7 @@ GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, strcplan.abort_callback = cpu_ctx->abort_callback;cplan.abort_callback_data = cpu_ctx->abort_callback_data;- ggml_graph_compute(cgraph, &cplan);- return true;+ return ggml_graph_compute(cgraph, &cplan);}GGML_CALL static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {@@ -791,8 +858,19 @@ static struct ggml_backend_i cpu_backend_i = {/* .graph_plan_compute = */ ggml_backend_cpu_graph_plan_compute,/* .graph_compute = */ ggml_backend_cpu_graph_compute,/* .supports_op = */ ggml_backend_cpu_supports_op,+ /* .offload_op = */ NULL,+ /* .event_new = */ NULL,+ /* .event_free = */ NULL,+ /* .event_record = */ NULL,+ /* .event_wait = */ NULL,+ /* .event_synchronize = */ NULL,};+static ggml_guid_t ggml_backend_cpu_guid(void) {+ static ggml_guid guid = { 0xaa, 0x67, 0xc7, 0x43, 0x96, 0xe6, 0xa3, 0x8a, 0xe3, 0xaf, 0xea, 0x92, 0x36, 0xbc, 0xfc, 0x89 };+ return &guid;+}+ggml_backend_t ggml_backend_cpu_init(void) {struct ggml_backend_cpu_context * ctx = malloc(sizeof(struct ggml_backend_cpu_context));if (ctx == NULL) {@@ -812,6 +890,7 @@ ggml_backend_t ggml_backend_cpu_init(void) {}*cpu_backend = (struct ggml_backend) {+ /* .guid = */ ggml_backend_cpu_guid(),/* .interface = */ cpu_backend_i,/* .context = */ ctx};@@ -819,7 +898,7 @@ ggml_backend_t ggml_backend_cpu_init(void) {}GGML_CALL bool ggml_backend_is_cpu(ggml_backend_t backend) {- return backend && backend->iface.get_name == ggml_backend_cpu_name;+ return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid());}void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {@@ -940,15 +1019,27 @@ static bool ggml_is_view_op(enum ggml_op op) {// scheduler-#define GGML_MAX_BACKENDS 16-#define GGML_MAX_SPLITS 256-#define GGML_MAX_SPLIT_INPUTS 16+#ifndef GGML_SCHED_MAX_BACKENDS+#define GGML_SCHED_MAX_BACKENDS 16+#endif++#ifndef GGML_SCHED_MAX_SPLITS+#define GGML_SCHED_MAX_SPLITS 2048+#endif++#ifndef GGML_SCHED_MAX_SPLIT_INPUTS+#define GGML_SCHED_MAX_SPLIT_INPUTS GGML_MAX_SRC+#endif++#ifndef GGML_SCHED_MAX_COPIES+#define GGML_SCHED_MAX_COPIES 4+#endifstruct ggml_backend_sched_split {int backend_id;int i_start;int i_end;- struct ggml_tensor * inputs[GGML_MAX_SPLIT_INPUTS];+ struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS];int n_inputs;// graph view of this splitstruct ggml_cgraph graph;@@ -956,27 +1047,37 @@ struct ggml_backend_sched_split {struct ggml_backend_sched {bool is_reset; // true if the scheduler has been reset since the last graph split+ bool is_alloc;int n_backends;- ggml_backend_t backends[GGML_MAX_BACKENDS];- ggml_backend_buffer_type_t bufts[GGML_MAX_BACKENDS];+ ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS];+ ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS];ggml_gallocr_t galloc;// hash keys of the nodes in the graphstruct ggml_hash_set hash_set;// hash valuesint * tensor_backend_id;- struct ggml_tensor * (* tensor_copies)[GGML_MAX_BACKENDS];+ struct ggml_tensor * (* tensor_copies)[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];- int * node_backend_ids; // [n_nodes]- int n_nodes;+ int * node_backend_ids; // [graph_size]+ int * leaf_backend_ids; // [graph_size]// copy of the graph with modified inputsstruct ggml_cgraph * graph;- struct ggml_backend_sched_split splits[GGML_MAX_SPLITS];+ // graph splits+ struct ggml_backend_sched_split * splits;int n_splits;+ int splits_capacity;++ // pipeline parallelism support+ int n_copies;+ int cur_copy;+ ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];+ struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS];+ int n_graph_inputs;struct ggml_context * ctx;@@ -984,17 +1085,16 @@ struct ggml_backend_sched {void * callback_eval_user_data;// align context_buffer to GGML_MEM_ALIGN- #ifdef _MSC_VER+#ifdef _MSC_VER__declspec(align(GGML_MEM_ALIGN))- #else+#else__attribute__((aligned(GGML_MEM_ALIGN)))- #endif- char context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)];+#endif+ char context_buffer[GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)];};-#define hash_id(node) ggml_hash_find_or_insert(sched->hash_set, node)-#define tensor_backend_id(node) sched->tensor_backend_id[hash_id(node)]-#define tensor_backend(node) (tensor_backend_id(node) == -1 ? NULL : sched->backends[tensor_backend_id(node)])+#define hash_id(tensor) ggml_hash_find_or_insert(sched->hash_set, tensor)+#define tensor_backend_id(tensor) sched->tensor_backend_id[hash_id(tensor)]// returns the priority of the backend, lower id is higher prioritystatic int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {@@ -1006,7 +1106,8 @@ static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backenreturn -1;}-static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) {+static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor) {+ ggml_backend_buffer_t buffer = tensor->buffer;if (buffer == NULL) {return -1;}@@ -1017,12 +1118,16 @@ static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, ggreturn i;}}- GGML_ASSERT(false && "tensor buffer type not supported by any backend");- return -1; // silence warning++ fprintf(stderr, "%s: error: no backend supports buffer type %s used in tensor %s\n",+ __func__, ggml_backend_buffer_name(buffer), tensor->name);+ GGML_ASSERT(false);++ return -1;}#if 0-static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug only+static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)#define GET_CAUSE(node) causes[hash_id(node)]#else@@ -1035,31 +1140,48 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st// TODO: use supports_op to check if the backend supports the op// assign pre-allocated nodes to their backend- // dst- int cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->buffer);- if (cur_backend != -1) {- SET_CAUSE(node, "1.dst");- return cur_backend;+ int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor);+ if (cur_backend_id != -1) {+ SET_CAUSE(tensor, "1.dst");+ return cur_backend_id;}+// view_srcif (tensor->view_src != NULL) {- cur_backend = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src->buffer);- if (cur_backend != -1) {- SET_CAUSE(node, "1.vsrc");- return cur_backend;+ cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src);+ if (cur_backend_id != -1) {+ SET_CAUSE(tensor, "1.vsrc");+ return cur_backend_id;}}++ // graph input+ if (tensor->flags & GGML_TENSOR_FLAG_INPUT) {+ cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU)+ SET_CAUSE(tensor, "1.inp");+ return cur_backend_id;+ }+// assign nodes that use weights to the backend of the weights+ // operations with weights are preferably run on the same backend as the weightsfor (int i = 0; i < GGML_MAX_SRC; i++) {const struct ggml_tensor * src = tensor->src[i];if (src == NULL) {continue;}if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {- int src_backend = ggml_backend_sched_backend_from_buffer(sched, src->buffer);- // operations with weights are always run on the same backend as the weights- SET_CAUSE(node, "1.wgt%d", i);- return src_backend;+ int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src);+ // check if a backend with higher prio wants to offload the op+ if (src_backend_id == sched->n_backends - 1) {+ for (int b = 0; b < src_backend_id; b++) {+ if (ggml_backend_offload_op(sched->backends[b], tensor)) {+ SET_CAUSE(tensor, "1.off");+ return b;+ }+ }+ }+ SET_CAUSE(tensor, "1.wgt%d", i);+ return src_backend_id;}}@@ -1094,7 +1216,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, strif (ggml_is_view_op(node->op)) {continue;}- ggml_backend_t tensor_backend = tensor_backend(node);+ ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, ggml_op_name(node->op), node->name,fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node));for (int j = 0; j < GGML_MAX_SRC; j++) {@@ -1102,7 +1224,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, strif (src == NULL) {continue;}- ggml_backend_t src_backend = tensor_backend(src);+ ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src);fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name,fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));}@@ -1119,6 +1241,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, strstatic void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {// reset splitssched->n_splits = 0;+ sched->n_graph_inputs = 0;sched->is_reset = false;struct ggml_init_params params = {@@ -1138,33 +1261,36 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg// pass 1: assign backends to ops with pre-allocated inputsfor (int i = 0; i < graph->n_leafs; i++) {struct ggml_tensor * leaf = graph->leafs[i];- if (tensor_backend_id(leaf) != -1) {+ int * leaf_backend_id = &tensor_backend_id(leaf);+ if (*leaf_backend_id != -1) {// do not overwrite user assignmentscontinue;}- tensor_backend_id(leaf) = ggml_backend_sched_backend_id_from_cur(sched, leaf);+ *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf);}for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- if (tensor_backend_id(node) != -1) {+ int * node_backend_id = &tensor_backend_id(node);+ if (*node_backend_id != -1) {// do not overwrite user assignmentscontinue;}- tensor_backend_id(node) = ggml_backend_sched_backend_id_from_cur(sched, node);+ *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node);// srcfor (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {continue;}- if (tensor_backend_id(src) == -1) {- tensor_backend_id(src) = ggml_backend_sched_backend_id_from_cur(sched, src);+ int * src_backend_id = &tensor_backend_id(src);+ if (*src_backend_id == -1) {+ *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src);}}}#ifdef DEBUG_PASS1- fprintf(stderr, "PASS 1 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+ fprintf(stderr, "PASS 1 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph);#endif// pass 2: expand current backend assignments@@ -1172,97 +1298,96 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg// expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)// thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops- // pass 2.1 expand gpu up++ // pass 2.2 expand gpu down{int cur_backend_id = -1;- for (int i = graph->n_nodes - 1; i >= 0; i--) {+ for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- int tensor_backend_id = tensor_backend_id(node);- if (tensor_backend_id != -1) {- if (tensor_backend_id == sched->n_backends - 1) {+ int * node_backend_id = &tensor_backend_id(node);+ if (*node_backend_id != -1) {+ if (*node_backend_id == sched->n_backends - 1) {// skip cpu (lowest prio backend)cur_backend_id = -1;} else {- cur_backend_id = tensor_backend_id;+ cur_backend_id = *node_backend_id;}} else {- tensor_backend_id(node) = cur_backend_id;- SET_CAUSE(node, "2.1");+ *node_backend_id = cur_backend_id;+ SET_CAUSE(node, "2.2");}}}-- // pass 2.2 expand gpu down+ // pass 2.1 expand gpu up{int cur_backend_id = -1;- for (int i = 0; i < graph->n_nodes; i++) {+ for (int i = graph->n_nodes - 1; i >= 0; i--) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- int tensor_backend_id = tensor_backend_id(node);- if (tensor_backend_id != -1) {- if (tensor_backend_id == sched->n_backends - 1) {+ int * node_backend_id = &tensor_backend_id(node);+ if (*node_backend_id != -1) {+ if (*node_backend_id == sched->n_backends - 1) {// skip cpu (lowest prio backend)cur_backend_id = -1;} else {- cur_backend_id = tensor_backend_id;+ cur_backend_id = *node_backend_id;}} else {- tensor_backend_id(node) = cur_backend_id;- SET_CAUSE(node, "2.2");+ *node_backend_id = cur_backend_id;+ SET_CAUSE(node, "2.1");}}}-- // pass 2.3 expand rest up+ // pass 2.4 expand rest down{int cur_backend_id = -1;- for (int i = graph->n_nodes - 1; i >= 0; i--) {+ for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- int tensor_backend_id = tensor_backend_id(node);- if (tensor_backend_id != -1) {- cur_backend_id = tensor_backend_id;+ int * node_backend_id = &tensor_backend_id(node);+ if (*node_backend_id != -1) {+ cur_backend_id = *node_backend_id;} else {- tensor_backend_id(node) = cur_backend_id;- SET_CAUSE(node, "2.3");+ *node_backend_id = cur_backend_id;+ SET_CAUSE(node, "2.4");}}}-- // pass 2.4 expand rest down+ // pass 2.3 expand rest up{int cur_backend_id = -1;- for (int i = 0; i < graph->n_nodes; i++) {+ for (int i = graph->n_nodes - 1; i >= 0; i--) {struct ggml_tensor * node = graph->nodes[i];if (ggml_is_view_op(node->op)) {continue;}- int tensor_backend_id = tensor_backend_id(node);- if (tensor_backend_id != -1) {- cur_backend_id = tensor_backend_id;+ int * node_backend_id = &tensor_backend_id(node);+ if (*node_backend_id != -1) {+ cur_backend_id = *node_backend_id;} else {- tensor_backend_id(node) = cur_backend_id;- SET_CAUSE(node, "2.4");+ *node_backend_id = cur_backend_id;+ SET_CAUSE(node, "2.3");}}}+#ifdef DEBUG_PASS2- fprintf(stderr, "PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+ fprintf(stderr, "PASS 2 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph);#endif// pass 3: assign backends to remaining src from dst and view_srcfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- int cur_backend_id = tensor_backend_id(node);- if (node->view_src != NULL && cur_backend_id == -1) {- cur_backend_id = tensor_backend_id(node) = tensor_backend_id(node->view_src);+ int * cur_backend_id = &tensor_backend_id(node);+ if (node->view_src != NULL && *cur_backend_id == -1) {+ *cur_backend_id = tensor_backend_id(node->view_src);SET_CAUSE(node, "3.vsrc");}for (int j = 0; j < GGML_MAX_SRC; j++) {@@ -1270,38 +1395,39 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggif (src == NULL) {continue;}- int src_backend_id = tensor_backend_id(src);- if (src_backend_id == -1) {+ int * src_backend_id = &tensor_backend_id(src);+ if (*src_backend_id == -1) {if (src->view_src != NULL) {// views are always on the same backend as the source- tensor_backend_id(src) = tensor_backend_id(src->view_src);+ *src_backend_id = tensor_backend_id(src->view_src);SET_CAUSE(src, "3.vsrc");} else {- tensor_backend_id(src) = cur_backend_id;+ *src_backend_id = *cur_backend_id;SET_CAUSE(src, "3.cur");}}}}#ifdef DEBUG_PASS3- fprintf(stderr, "PASS 3 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+ fprintf(stderr, "PASS 3 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph);#endif// pass 4: split graph, find tensors that need to be copied{- int cur_split = 0;+ int i_split = 0;+ struct ggml_backend_sched_split * split = &sched->splits[0];// find the backend of the first split, skipping view opsfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];if (!ggml_is_view_op(node->op)) {- sched->splits[0].backend_id = tensor_backend_id(node);+ split->backend_id = tensor_backend_id(node);break;}}- sched->splits[0].i_start = 0;- sched->splits[0].n_inputs = 0;- memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK- int cur_backend_id = sched->splits[0].backend_id;+ split->i_start = 0;+ split->n_inputs = 0;+ memset(split->inputs, 0, sizeof(split->inputs)); //HACK+ int cur_backend_id = split->backend_id;for (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];@@ -1309,18 +1435,54 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggcontinue;}- int tensor_backend_id = tensor_backend_id(node);+ const int node_backend_id = tensor_backend_id(node);- GGML_ASSERT(tensor_backend_id != -1); // all nodes should be assigned by now+ GGML_ASSERT(node_backend_id != -1); // all nodes should be assigned by now- if (tensor_backend_id != cur_backend_id) {- sched->splits[cur_split].i_end = i;- cur_split++;- GGML_ASSERT(cur_split < GGML_MAX_SPLITS);- sched->splits[cur_split].backend_id = tensor_backend_id;- sched->splits[cur_split].i_start = i;- sched->splits[cur_split].n_inputs = 0;- cur_backend_id = tensor_backend_id;+ // check if we should start a new split based on the sources of the current node+ bool need_new_split = false;+ if (node_backend_id == cur_backend_id && split->n_inputs > 0) {+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ continue;+ }+ // check if a weight is on a different backend+ // by starting a new split, the memory of the previously offloaded weights can be reused+ if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {+ int src_backend_id = tensor_backend_id(src);+ if (src_backend_id != -1 && src_backend_id != cur_backend_id) {+ need_new_split = true;+ break;+ }+ }+ // check if the split has too many inputs+ if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) {+ const size_t id = hash_id(src);+ int src_backend_id = sched->tensor_backend_id[id];+ if (src_backend_id != cur_backend_id && sched->tensor_copies[hash_id(src)][cur_backend_id][0] == NULL) {+ //printf("starting new split because of too many inputs: node %s, input %s\n", node->name, src->name);+ need_new_split = true;+ break;+ }+ }+ }+ }++ if (node_backend_id != cur_backend_id || need_new_split) {+ split->i_end = i;+ i_split++;+ if (i_split >= sched->splits_capacity) {+ sched->splits_capacity *= 2;+ sched->splits = realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));+ GGML_ASSERT(sched->splits != NULL);+ }+ GGML_ASSERT(i_split < GGML_SCHED_MAX_SPLITS);+ split = &sched->splits[i_split];+ split->backend_id = node_backend_id;+ split->i_start = i;+ split->n_inputs = 0;+ cur_backend_id = node_backend_id;}// find inputs that are not on the same backend@@ -1329,83 +1491,84 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggif (src == NULL) {continue;}- int src_backend_id = tensor_backend_id(src);++ const int src_backend_id = tensor_backend_id(src);assert(src_backend_id != -1); // all inputs should be assigned by now- if (src_backend_id != tensor_backend_id) {- // create a copy of the input in the split's backend- size_t id = hash_id(src);- if (sched->tensor_copies[id][cur_backend_id] == NULL) {- ggml_backend_t backend = sched->backends[cur_backend_id];- struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);- ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name);- sched->tensor_copies[id][cur_backend_id] = tensor_copy;- tensor_backend_id(tensor_copy) = cur_backend_id;- SET_CAUSE(tensor_copy, "4.cpy");+ if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) {+ size_t id = hash_id(src);+ if (sched->tensor_copies[id][src_backend_id][0] == NULL) {+ ggml_backend_t backend = sched->backends[src_backend_id];+ for (int c = 0; c < sched->n_copies; c++) {+ struct ggml_tensor * tensor_copy;+ if (c == sched->cur_copy) {+ tensor_copy = src; // use the original tensor as the current copy+ } else {+ tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);+ ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);+ }+ if (sched->n_copies > 1) {+ ggml_set_input(tensor_copy);+ ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor+ }+ sched->tensor_copies[id][src_backend_id][c] = tensor_copy;+ SET_CAUSE(tensor_copy, "4.cpy");+ }+ int n_graph_inputs = sched->n_graph_inputs++;+ GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);+ sched->graph_inputs[n_graph_inputs] = src;+ }+ }- int n_inputs = sched->splits[cur_split].n_inputs++;- GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS);- sched->splits[cur_split].inputs[n_inputs] = src;+ if (src_backend_id != node_backend_id) {+ // create a copy of the input in the split's backend+ const size_t id = hash_id(src);+ if (sched->tensor_copies[id][cur_backend_id][0] == NULL) {+ ggml_backend_t backend = sched->backends[cur_backend_id];+ for (int c = 0; c < sched->n_copies; c++) {+ struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);+ ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);+ if (sched->n_copies > 1) {+ ggml_set_input(tensor_copy);+ ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor+ }+ sched->tensor_copies[id][cur_backend_id][c] = tensor_copy;+ SET_CAUSE(tensor_copy, "4.cpy");+ }+ int n_inputs = split->n_inputs++;+ GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);+ split->inputs[n_inputs] = src;}- node->src[j] = sched->tensor_copies[id][cur_backend_id];+ node->src[j] = sched->tensor_copies[id][cur_backend_id][sched->cur_copy];}}}- sched->splits[cur_split].i_end = graph->n_nodes;- sched->n_splits = cur_split + 1;+ split->i_end = graph->n_nodes;+ sched->n_splits = i_split + 1;}#ifdef DEBUG_PASS4- fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);-#endif--#ifndef NDEBUG- // sanity check: all sources should have the same backend as the node- for (int i = 0; i < graph->n_nodes; i++) {- struct ggml_tensor * node = graph->nodes[i];- ggml_backend_t tensor_backend = tensor_backend(node);- if (tensor_backend == NULL) {- fprintf(stderr, "!!!!!!! %s has no backend\n", node->name);- }- if (node->view_src != NULL && tensor_backend != tensor_backend(node->view_src)) {- fprintf(stderr, "!!!!!!! %s has backend %s, view_src %s has backend %s\n",- node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL",- node->view_src->name, tensor_backend(node->view_src) ? ggml_backend_name(tensor_backend(node->view_src)) : "NULL");- }- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * src = node->src[j];- if (src == NULL) {- continue;- }- ggml_backend_t src_backend = tensor_backend(src);- if (src_backend != tensor_backend /* && src_backend != NULL */) {- fprintf(stderr, "!!!! %s has backend %s, src %d (%s) has backend %s\n",- node->name, tensor_backend ? ggml_backend_name(tensor_backend) : "NULL",- j, src->name, src_backend ? ggml_backend_name(src_backend) : "NULL");- }- if (src->view_src != NULL && src_backend != tensor_backend(src->view_src)) {- fprintf(stderr, "!!!!!!! [src] %s has backend %s, view_src %s has backend %s\n",- src->name, src_backend ? ggml_backend_name(src_backend) : "NULL",- src->view_src->name, tensor_backend(src->view_src) ? ggml_backend_name(tensor_backend(src->view_src)) : "NULL");- }- }- }- fflush(stderr);+ fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); ggml_backend_sched_print_assignments(sched, graph);#endif// create copies of the graph for each split- // FIXME: avoid this copy, pass split inputs to ggml_gallocr_alloc_graph_n in some other way- struct ggml_cgraph * graph_copy = ggml_new_graph_custom(sched->ctx, graph->n_nodes + sched->n_splits*GGML_MAX_SPLIT_INPUTS, false);+ // TODO: avoid this copy+ struct ggml_cgraph * graph_copy = ggml_new_graph_custom(sched->ctx, graph->n_nodes + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2, false);for (int i = 0; i < sched->n_splits; i++) {struct ggml_backend_sched_split * split = &sched->splits[i];split->graph = ggml_graph_view(graph, split->i_start, split->i_end);+ // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the splitfor (int j = 0; j < split->n_inputs; j++) {+ assert(graph_copy->size > (graph_copy->n_nodes + 1));+struct ggml_tensor * input = split->inputs[j];- struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split->backend_id];+ const size_t input_id = hash_id(input);+ struct ggml_tensor * input_cpy = sched->tensor_copies[input_id][split->backend_id][sched->cur_copy];// add a dependency to the input source so that it is not freed before the copy is donestruct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input);- sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(input);+ input_dep->src[0] = input;+ sched->node_backend_ids[graph_copy->n_nodes] = sched->tensor_backend_id[input_id];graph_copy->nodes[graph_copy->n_nodes++] = input_dep;// add a dependency to the input copy so that it is allocated at the start of the split@@ -1414,22 +1577,61 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg}for (int j = split->i_start; j < split->i_end; j++) {+ assert(graph_copy->size > graph_copy->n_nodes);sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];}}++ if (sched->n_copies > 1) {+ // add input copies as leafs so that they are allocated first+ for (int i = 0; i < sched->n_graph_inputs; i++) {+ struct ggml_tensor * input = sched->graph_inputs[i];+ size_t id = hash_id(input);+ int backend_id = tensor_backend_id(input);+ for (int c = 0; c < sched->n_copies; c++) {+ struct ggml_tensor * input_cpy = sched->tensor_copies[id][backend_id][c];+ sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;+ graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;+ }+ }++ for (int i = 0; i < sched->n_splits; i++) {+ struct ggml_backend_sched_split * split = &sched->splits[i];+ int backend_id = split->backend_id;+ for (int j = 0; j < split->n_inputs; j++) {+ struct ggml_tensor * input = split->inputs[j];+ size_t id = hash_id(input);+ for (int c = 0; c < sched->n_copies; c++) {+ struct ggml_tensor * input_cpy = sched->tensor_copies[id][backend_id][c];+ sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;+ graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;+ }+ }+ }+ }++ // add leafs from the original graph+ for (int i = 0; i < graph->n_leafs; i++) {+ struct ggml_tensor * leaf = graph->leafs[i];+ sched->leaf_backend_ids[graph_copy->n_leafs] = tensor_backend_id(leaf);+ graph_copy->leafs[graph_copy->n_leafs++] = leaf;+ }+sched->graph = graph_copy;}static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {- // ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids);+ // allocate graphif (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) {+ // the re-allocation may cause the split inputs to be moved to a different address+ ggml_backend_sched_synchronize(sched);#ifndef NDEBUG- fprintf(stderr, "ggml_backend_sched: failed to allocate graph, reserving\n");+ fprintf(stderr, "%s: failed to allocate graph, reserving\n", __func__);#endif- ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids);+ ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids, sched->leaf_backend_ids);if (!ggml_gallocr_alloc_graph(sched->galloc, sched->graph)) {- fprintf(stderr, "ggml_backend_sched: failed to allocate graph\n");+ fprintf(stderr, "%s: failed to allocate graph\n", __func__);return false;}}@@ -1437,10 +1639,7 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {return true;}-static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {- uint64_t copy_us[GGML_MAX_BACKENDS] = {0};- uint64_t compute_us[GGML_MAX_BACKENDS] = {0};-+static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {struct ggml_backend_sched_split * splits = sched->splits;for (int i = 0; i < sched->n_splits; i++) {@@ -1449,33 +1648,35 @@ static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {ggml_backend_t split_backend = sched->backends[split_backend_id];// copy the input tensors to the split backend- uint64_t copy_start_us = ggml_time_us();for (int j = 0; j < split->n_inputs; j++) {+ ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[j]);struct ggml_tensor * input = split->inputs[j];- struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split_backend_id];-- GGML_ASSERT(input->buffer != NULL);- GGML_ASSERT(input_cpy->buffer != NULL);+ struct ggml_tensor * input_cpy = sched->tensor_copies[hash_id(input)][split_backend_id][sched->cur_copy];- ggml_backend_tensor_copy_async(split_backend, input, input_cpy);+ if (input->flags & GGML_TENSOR_FLAG_INPUT) {+ // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {+ ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);+ } else {+ ggml_backend_synchronize(split_backend);+ }+ ggml_backend_tensor_copy(input, input_cpy);+ } else {+ // wait for the split backend to finish using the input before overwriting it+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {+ ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);+ } else {+ ggml_backend_synchronize(split_backend);+ }+ ggml_backend_tensor_copy_async(input_backend, split_backend, input, input_cpy);+ }}- //ggml_backend_synchronize(split_backend); // necessary to measure copy time- int64_t copy_end_us = ggml_time_us();- copy_us[split_backend_id] += copy_end_us - copy_start_us;--#if 0- char split_filename[GGML_MAX_NAME];- snprintf(split_filename, GGML_MAX_NAME, "split_%i_%s.dot", i, ggml_backend_name(split_backend));- ggml_graph_dump_dot(split->graph, NULL, split_filename);-#endif-- uint64_t compute_start_us = ggml_time_us();if (!sched->callback_eval) {- if (!ggml_backend_graph_compute(split_backend, &split->graph)) {- return false;+ enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);+ if (ec != GGML_STATUS_SUCCESS) {+ return ec;}- //ggml_backend_synchronize(split_backend); // necessary to measure compute time} else {// similar to ggml_backend_compare_graph_backendfor (int j0 = 0; j0 < split->graph.n_nodes; j0++) {@@ -1494,10 +1695,14 @@ static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);- if (!ggml_backend_graph_compute(split_backend, &gv)) {- return false;+ enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv);+ if (ec != GGML_STATUS_SUCCESS) {+ return ec;}+ // TODO: pass backend to the callback, then the user can decide if they want to synchronize+ ggml_backend_synchronize(split_backend);+if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) {break;}@@ -1505,39 +1710,58 @@ static bool ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {j0 = j1;}}- uint64_t compute_end_us = ggml_time_us();- compute_us[split_backend_id] += compute_end_us - compute_start_us;- }-#if 0- // per-backend timings- fprintf(stderr, "sched_compute_splits times (%d splits):\n", sched->n_splits);- for (int i = 0; i < sched->n_backends; i++) {- if (copy_us[i] > 0 || compute_us[i] > 0) {- fprintf(stderr, "\t%5.5s: %lu us copy, %lu us compute\n", ggml_backend_name(sched->backends[i]), copy_us[i], compute_us[i]);+ // record the event of this copy+ if (split->n_inputs > 0) {+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {+ ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy]);+ }}}-#endif- return true;+ sched->cur_copy = (sched->cur_copy + 1) % sched->n_copies;++ return GGML_STATUS_SUCCESS;}-ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size) {+ggml_backend_sched_t ggml_backend_sched_new(+ ggml_backend_t * backends,+ ggml_backend_buffer_type_t * bufts,+ int n_backends,+ size_t graph_size,+ bool parallel) {GGML_ASSERT(n_backends > 0);- GGML_ASSERT(n_backends <= GGML_MAX_BACKENDS);+ GGML_ASSERT(n_backends <= GGML_SCHED_MAX_BACKENDS);+ GGML_ASSERT(ggml_backend_is_cpu(backends[n_backends - 1])); // last backend must be CPUstruct ggml_backend_sched * sched = calloc(sizeof(struct ggml_backend_sched), 1);// initialize hash table- sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);+ sched->hash_set = ggml_hash_set_new(graph_size);sched->tensor_backend_id = calloc(sizeof(sched->tensor_backend_id[0]), sched->hash_set.size);sched->tensor_copies = calloc(sizeof(sched->tensor_copies[0]), sched->hash_set.size);- sched->node_backend_ids = calloc(sizeof(sched->node_backend_ids[0]), graph_size);++ const size_t nodes_size = graph_size + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2;+ sched->node_backend_ids = calloc(sizeof(sched->node_backend_ids[0]), nodes_size);+ sched->leaf_backend_ids = calloc(sizeof(sched->leaf_backend_ids[0]), nodes_size);sched->n_backends = n_backends;- for (int i = 0; i < n_backends; i++) {- sched->backends[i] = backends[i];- sched->bufts[i] = bufts ? bufts[i] : ggml_backend_get_default_buffer_type(backends[i]);++ sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1;++ const int initial_splits_capacity = 16;+ sched->splits = calloc(sizeof(sched->splits[0]), initial_splits_capacity);+ sched->splits_capacity = initial_splits_capacity;++ for (int b = 0; b < n_backends; b++) {+ sched->backends[b] = backends[b];+ sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]);+ GGML_ASSERT(ggml_backend_buft_supports_backend(sched->bufts[b], backends[b]));+ if (sched->n_copies > 1) {+ for (int c = 0; c < sched->n_copies; c++) {+ sched->events[b][c] = ggml_backend_event_new(backends[b]);+ }+ }}sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends);@@ -1551,12 +1775,19 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {if (sched == NULL) {return;}+ for (int b = 0; b < sched->n_backends; b++) {+ for (int c = 0; c < sched->n_copies; c++) {+ ggml_backend_event_free(sched->events[b][c]);+ }+ }ggml_gallocr_free(sched->galloc);ggml_free(sched->ctx);+ free(sched->splits);free(sched->hash_set.keys);free(sched->tensor_backend_id);free(sched->tensor_copies);free(sched->node_backend_ids);+ free(sched->leaf_backend_ids);free(sched);}@@ -1568,38 +1799,65 @@ void ggml_backend_sched_reset(ggml_backend_sched_t sched) {memset(sched->tensor_copies, 0, sizeof(sched->tensor_copies[0]) * hash_size);sched->is_reset = true;+ sched->is_alloc = false;}bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {+ GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes);+ggml_backend_sched_split_graph(sched, measure_graph);- if (!ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids)) {+ // TODO: extract this to a separate function+ if (!ggml_gallocr_reserve_n(sched->galloc, sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) {return false;}ggml_backend_sched_reset(sched);+ ggml_backend_sched_synchronize(sched);+return true;}-bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {- GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);-- if (!sched->is_reset) {- ggml_backend_sched_reset(sched);- }+bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+ GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes);ggml_backend_sched_split_graph(sched, graph);+if (!ggml_backend_sched_alloc_splits(sched)) {return false;}- if (!ggml_backend_sched_compute_splits(sched)) {- return false;- }+ sched->is_alloc = true;return true;}+enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+ enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph);+ ggml_backend_sched_synchronize(sched);+ return err;+}++enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {+ if (!sched->is_reset && !sched->is_alloc) {+ ggml_backend_sched_reset(sched);+ }++ if (!sched->is_alloc) {+ if (!ggml_backend_sched_alloc_graph(sched, graph)) {+ return GGML_STATUS_ALLOC_FAILED;+ }+ }++ return ggml_backend_sched_compute_splits(sched);+}++void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) {+ for (int i = 0; i < sched->n_backends; i++) {+ ggml_backend_synchronize(sched->backends[i]);+ }+}+void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) {sched->callback_eval = callback;sched->callback_eval_user_data = user_data;@@ -1609,19 +1867,24 @@ int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {return sched->n_splits;}+int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched) {+ return sched->n_copies;+}+size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) {int backend_index = ggml_backend_sched_backend_id(sched, backend);GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);+return ggml_gallocr_get_buffer_size(sched->galloc, backend_index);}-void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {+void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {int backend_index = ggml_backend_sched_backend_id(sched, backend);GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);tensor_backend_id(node) = backend_index;}-ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {+ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {int backend_index = tensor_backend_id(node);if (backend_index == -1) {return NULL;diff --git a/llama.cpp/ggml-backend.h b/llama.cpp/ggml-backend.hindex 9ea70ead..ba9b1518 100644--- a/llama.cpp/ggml-backend.h+++ b/llama.cpp/ggml-backend.h@@ -9,6 +9,7 @@ extern "C" {typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;typedef struct ggml_backend_buffer * ggml_backend_buffer_t;+ typedef struct ggml_backend_event * ggml_backend_event_t;typedef struct ggml_backend * ggml_backend_t;typedef void * ggml_backend_graph_plan_t;@@ -49,7 +50,7 @@ extern "C" {// Backend//-+ GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);GGML_API const char * ggml_backend_name(ggml_backend_t backend);GGML_API void ggml_backend_free(ggml_backend_t backend);@@ -66,16 +67,30 @@ extern "C" {GGML_API void ggml_backend_synchronize(ggml_backend_t backend);- GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create (ggml_backend_t backend, struct ggml_cgraph * cgraph);+ GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);+ GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);- GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);- GGML_API void ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan);- GGML_API bool ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);- GGML_API bool ggml_backend_supports_op (ggml_backend_t backend, const struct ggml_tensor * op);+ GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);+ GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);+ GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);+ GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);+ GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);// tensor copy between different backendsGGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);- GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst); // automatic fallback to sync copy++ // asynchronous copy+ // the copy is performed after all the currently queued operations in backend_src+ // backend_dst will wait for the copy to complete before performing other operations+ // automatic fallback to sync copy if async is not supported+ GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);++ // events+ GGML_API ggml_backend_event_t ggml_backend_event_new (ggml_backend_t backend);+ GGML_API void ggml_backend_event_free (ggml_backend_event_t event);+ GGML_API void ggml_backend_event_record (ggml_backend_event_t event);+ GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);+ GGML_API void ggml_backend_event_wait (ggml_backend_t backend, ggml_backend_event_t event); // wait async on event//// CPU backend@@ -122,27 +137,31 @@ extern "C" {/*Example usage:- sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, num_backends);- // sched is initialized with measure allocators and cannot be used until allocated with a measure graph+ // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be asigned+ // preferrably to run on the same backend as the buffer+ ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);- // initialize buffers from a measure graph- measure_graph = build_graph(sched); // use the allocr to allocate inputs as needed+ sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false);- // in build_graph:- build_graph(...) {- // manually assign nodes to a backend (optional, should not be needed in most cases)- struct ggml_tensor * node = ggml_mul_mat(ctx, ...);- ggml_backend_sched_set_node_backend(sched, node, backend_gpu);- }+ // initialize buffers from a max size graph (optional)+ reserve_graph = build_graph(sched, max_batch_size);- // allocate backend buffers from measure graph- ggml_backend_sched_init_measure(sched, measure_graph);+ // manually assign nodes to a backend (optional, should not be needed in most cases)+ struct ggml_tensor * node = ggml_mul_mat(ctx, ...);+ ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);- // the scheduler is now ready to compute graphs+ ggml_backend_sched_reserve(sched, reserve_graph);// computegraph = build_graph(sched);ggml_backend_sched_graph_compute(sched, graph);++ // if there are graph inputs:+ ggml_backend_sched_reset(sched);+ ggml_backend_sched_alloc_graph(sched, graph);+ ggml_backend_tensor_set(input_tensor, ...);+ ggml_backend_sched_graph_compute(sched, graph);+ }*/struct ggml_backend_sched;@@ -157,26 +176,32 @@ extern "C" {typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);// Initialize a backend scheduler- GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size);- GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);+ GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);+ GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);+// Initialize backend buffers from a measure graph- GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);+ GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);+// Get the number of splits of the last graph- GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);+ GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);+ GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);- GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);+ GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);- GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);- GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);+ GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);+ GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);// Allocate and compute graph on the backend scheduler- GGML_API bool ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);+ GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);+ GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);+ GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);+ GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);// Reset all assignments and allocators - must be called before changing the node backends- GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);+ GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);// Set a callback to be called for each resulting node during graph compute- GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);+ GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);//// Utilsdiff --git a/llama.cpp/ggml-common.h b/llama.cpp/ggml-common.hnew file mode 100644index 00000000..0257c928--- /dev/null+++ b/llama.cpp/ggml-common.h@@ -0,0 +1,1830 @@+#ifndef GGML_COMMON_DECL++#if defined(GGML_COMMON_DECL_C)+#include <stdint.h>++typedef uint16_t ggml_half;+typedef uint32_t ggml_half2;++#define GGML_COMMON_AGGR++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_METAL)+#include <metal_stdlib>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_CUDA)+#include <cuda_fp16.h>+#include <cstdint>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR data++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_HIP)+#include <hip/hip_fp16.h>+#include <cstdint>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR data++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_SYCL)+#include <sycl/half_type.hpp>+#include <cstdint>++typedef sycl::half ggml_half;+typedef sycl::half2 ggml_half2;++#define GGML_COMMON_AGGR data++#define GGML_COMMON_DECL+#endif++#if defined(GGML_COMMON_DECL)++#ifndef __cplusplus+#ifndef static_assert+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)+#define static_assert(cond, msg) _Static_assert(cond, msg)+#else+#define static_assert(cond, msg) struct global_scope_noop_trick+#endif+#endif+#endif // __cplusplus++// QK = number of values after dequantization+// QK_K = super-block size++#ifdef GGML_QKK_64+#define QK_K 64+#define K_SCALE_SIZE 4+#else+#define QK_K 256+#define K_SCALE_SIZE 12+#endif // GGML_QKK_64++#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL)+// QR = QK / number of values before dequantization+// QI = number of 32 bit integers before dequantization++#define QI4_0 (QK4_0 / (4 * QR4_0))+#define QR4_0 2++#define QI4_1 (QK4_1 / (4 * QR4_1))+#define QR4_1 2++#define QI5_0 (QK5_0 / (4 * QR5_0))+#define QR5_0 2++#define QI5_1 (QK5_1 / (4 * QR5_1))+#define QR5_1 2++#define QI8_0 (QK8_0 / (4 * QR8_0))+#define QR8_0 1++#define QI8_1 (QK8_1 / (4 * QR8_1))+#define QR8_1 1++#define QI2_K (QK_K / (4*QR2_K))+#define QR2_K 4++#define QI3_K (QK_K / (4*QR3_K))+#define QR3_K 4++#define QI4_K (QK_K / (4*QR4_K))+#define QR4_K 2++#define QI5_K (QK_K / (4*QR5_K))+#define QR5_K 2++#define QI6_K (QK_K / (4*QR6_K))+#define QR6_K 2++#define QI2_XXS (QK_K / (4*QR2_XXS))+#define QR2_XXS 8++#define QI2_XS (QK_K / (4*QR2_XS))+#define QR2_XS 8++#define QI2_S (QK_K / (4*QR2_S))+#define QR2_S 8++#define QI3_XXS (QK_K / (4*QR3_XXS))+#define QR3_XXS 8++#define QI3_XS (QK_K / (4*QR3_XS))+#define QR3_XS 8++#define QI1_S (QK_K / (4*QR1_S))+#define QR1_S 8++#define QI4_NL (QK4_NL / (4*QR4_NL))+#define QR4_NL 2++#if QK_K == 64+#define QI4_XS QI4_NL+#define QR4_XS QR4_NL+#else+#define QI4_XS (QK_K / (4*QR4_XS))+#define QR4_XS 8+#endif++#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP++#define QK4_0 32+typedef struct {+ ggml_half d; // delta+ uint8_t qs[QK4_0 / 2]; // nibbles / quants+} block_q4_0;+static_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, "wrong q4_0 block size/padding");++#define QK4_1 32+typedef struct {+ union {+ struct {+ ggml_half d; // delta+ ggml_half m; // min+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t qs[QK4_1 / 2]; // nibbles / quants+} block_q4_1;+static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");++#define QK5_0 32+typedef struct {+ ggml_half d; // delta+ uint8_t qh[4]; // 5-th bit of quants+ uint8_t qs[QK5_0 / 2]; // nibbles / quants+} block_q5_0;+static_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");++#define QK5_1 32+typedef struct {+ union {+ struct {+ ggml_half d; // delta+ ggml_half m; // min+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t qh[4]; // 5-th bit of quants+ uint8_t qs[QK5_1 / 2]; // nibbles / quants+} block_q5_1;+static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");++#define QK8_0 32+typedef struct {+ ggml_half d; // delta+ int8_t qs[QK8_0]; // quants+} block_q8_0;+static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding");++#define QK8_1 32+typedef struct {+ union {+ struct {+ ggml_half d; // delta+ ggml_half s; // d * sum(qs[i])+ } GGML_COMMON_AGGR;+ ggml_half2 ds;+ };+ int8_t qs[QK8_1]; // quants+} block_q8_1;+static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding");++//+// Super-block quantization structures+//++// 2-bit quantization+// weight is represented as x = a * q + b+// 16 blocks of 16 elements each+// Effectively 2.625 bits per weight+typedef struct {+ uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits+ uint8_t qs[QK_K/4]; // quants+ union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+} block_q2_K;+static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");++// 3-bit quantization+// weight is represented as x = a * q+// 16 blocks of 16 elements each+// Effectively 3.4375 bits per weight+#ifdef GGML_QKK_64+typedef struct {+ uint8_t hmask[QK_K/8]; // quants - high bit+ uint8_t qs[QK_K/4]; // quants - low 2 bits+ uint8_t scales[2];+ ggml_half d; // super-block scale+} block_q3_K;+static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding");+#else+typedef struct {+ uint8_t hmask[QK_K/8]; // quants - high bit+ uint8_t qs[QK_K/4]; // quants - low 2 bits+ uint8_t scales[12]; // scales, quantized with 6 bits+ ggml_half d; // super-block scale+} block_q3_K;+static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");+#endif++// 4-bit quantization+// 8 blocks of 32 elements each+// weight is represented as x = a * q + b+// Effectively 4.5 bits per weight+#ifdef GGML_QKK_64+typedef struct {+ ggml_half d[2]; // super-block scales/mins+ uint8_t scales[2]; // 4-bit block scales/mins+ uint8_t qs[QK_K/2]; // 4--bit quants+} block_q4_K;+static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + QK_K/2 + 2, "wrong q4_K block size/padding");+#else+typedef struct {+ union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits+ uint8_t qs[QK_K/2]; // 4--bit quants+} block_q4_K;+static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");+#endif++// 5-bit quantization+// 8 blocks of 32 elements each+// weight is represented as x = a * q + b+// Effectively 5.5 bits per weight+#ifdef GGML_QKK_64+typedef struct {+ ggml_half d; // super-block scale+ int8_t scales[QK_K/16]; // 8-bit block scales+ uint8_t qh[QK_K/8]; // quants, high bit+ uint8_t qs[QK_K/2]; // quants, low 4 bits+} block_q5_K;+static_assert(sizeof(block_q5_K) == sizeof(ggml_half) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding");+#else+typedef struct {+ union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits+ uint8_t qh[QK_K/8]; // quants, high bit+ uint8_t qs[QK_K/2]; // quants, low 4 bits+} block_q5_K;+static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");+#endif++// 6-bit quantization+// weight is represented as x = a * q+// 16 blocks of 16 elements each+// Effectively 6.5625 bits per weight+typedef struct {+ uint8_t ql[QK_K/2]; // quants, lower 4 bits+ uint8_t qh[QK_K/4]; // quants, upper 2 bits+ int8_t scales[QK_K/16]; // scales, quantized with 8 bits+ ggml_half d; // super-block scale+} block_q6_K;+static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");++// This is only used for intermediate quantization and dot products+typedef struct {+ float d; // delta+ int8_t qs[QK_K]; // quants+ int16_t bsums[QK_K/16]; // sum of quants in groups of 16+} block_q8_K;+static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");++// (Almost) "true" 2-bit quantization.+// Due to the need to use blocks as per ggml design, it ends up using+// 2.0625 bpw because of the 16-bit scale for each block of 256.+typedef struct {+ ggml_half d;+ uint16_t qs[QK_K/8];+} block_iq2_xxs;+static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");++// 2.3125 bpw quants+typedef struct {+ ggml_half d;+ uint16_t qs[QK_K/8];+ uint8_t scales[QK_K/32];+} block_iq2_xs;+static_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");++// 2.5625 bpw quants+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/4];+ uint8_t qh[QK_K/32];+ uint8_t scales[QK_K/32];+} block_iq2_s;+static_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding");++// (Almost) "true" 3-bit quantization.+// Due to the need to use blocks as per ggml design, it ends up using+// 3.0625 bpw because of the 16-bit scale for each block of 256.+typedef struct {+ ggml_half d;+ uint8_t qs[3*QK_K/8];+} block_iq3_xxs;+static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");++// 3.4375 bpw+#if QK_K == 64+#define IQ3S_N_SCALE 2+#else+#define IQ3S_N_SCALE QK_K/64+#endif+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/4];+ uint8_t qh[QK_K/32];+ uint8_t signs[QK_K/8];+ uint8_t scales[IQ3S_N_SCALE];+} block_iq3_s;+static_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding");++typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/8];+ uint16_t qh[QK_K/32];+} block_iq1_s;+static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");++// Non-linear quants+#define QK4_NL 32+typedef struct {+ ggml_half d;+ uint8_t qs[QK4_NL/2];+} block_iq4_nl;+static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding");++#if QK_K == 64+#define block_iq4_xs block_iq4_nl+#else+typedef struct {+ ggml_half d;+ uint16_t scales_h;+ uint8_t scales_l[QK_K/64];+ uint8_t qs[QK_K/2];+} block_iq4_xs;+static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");+#endif++#endif // GGML_COMMON_DECL+#endif // GGML_COMMON_DECL++////////////////////////////////////////////////////////////////////////////////++#ifndef GGML_COMMON_IMPL++#if defined(GGML_COMMON_IMPL_C)+#include <stdint.h>++#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_METAL)+#include <metal_stdlib>++#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP)+#include <cstdint>++#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_SYCL)+#include <cstdint>++#define GGML_TABLE_BEGIN(type, name, size) static dpct::global_memory<const type, 1> name(sycl::range<1>(size), {+#define GGML_TABLE_END() });++#define GGML_COMMON_IMPL+#endif++#if defined(GGML_COMMON_IMPL)++GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)+ 1, 2, 4, 8, 16, 32, 64, 128+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)+ 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,+ 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,+ 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,+ 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,+ 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,+ 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,+ 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,+ 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,+GGML_TABLE_END()++//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)+ 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,+ 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,+ 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,+ 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,+ 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,+ 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,+ 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,+ 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,+ 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,+ 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,+ 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,+ 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,+ 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,+ 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,+ 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,+ 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,+ 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,+ 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,+ 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,+ 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,+ 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,+ 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,+ 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,+ 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,+ 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,+ 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,+ 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,+ 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,+ 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,+ 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,+ 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,+ 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,+GGML_TABLE_END()+//#endif+++GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,+ 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,+ 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,+ 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,+ 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,+ 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,+ 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,+ 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,+ 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,+ 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,+ 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,+ 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,+ 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,+ 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,+ 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,+ 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,+ 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,+ 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,+ 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,+ 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,+ 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,+ 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,+ 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,+ 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,+ 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,+ 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,+ 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,+ 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,+ 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,+ 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,+ 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,+ 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,+ 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,+ 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,+ 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,+ 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,+ 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,+ 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,+ 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,+ 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,+ 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,+ 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,+ 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,+ 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,+ 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,+ 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,+ 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,+ 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,+ 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,+ 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,+ 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,+ 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,+ 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,+ 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,+ 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,+ 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,+ 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,+ 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,+ 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,+ 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,+ 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,+ 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,+ 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,+ 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,+ 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,+ 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,+ 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,+ 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,+ 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,+ 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,+ 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,+ 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,+ 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,+ 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,+ 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,+ 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,+ 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,+ 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,+ 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,+ 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,+ 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,+ 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,+ 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,+ 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,+ 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,+ 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,+ 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,+ 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,+ 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,+ 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,+ 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,+ 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,+ 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,+ 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,+ 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,+ 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,+ 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,+ 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,+ 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,+ 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,+ 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,+ 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,+ 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,+ 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,+ 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,+ 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,+ 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,+ 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,+ 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,+ 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,+ 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,+ 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,+ 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,+ 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,+ 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,+ 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,+ 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,+ 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,+ 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,+ 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,+ 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,+ 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,+ 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,+ 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,+ 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,+ 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,+ 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,+ 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,+ 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,+ 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,+ 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,+ 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,+ 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,+ 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,+ 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,+ 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,+ 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,+ 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,+ 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,+ 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,+ 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,+ 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,+ 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,+ 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,+ 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,+ 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,+ 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,+ 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,+ 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,+ 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,+ 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,+ 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,+ 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,+ 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,+ 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,+ 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,+ 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,+ 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,+ 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,+ 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,+ 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,+ 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,+ 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,+ 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,+ 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,+ 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,+ 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,+ 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,+ 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,+ 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,+ 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,+ 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,+ 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,+ 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,+ 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,+ 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,+ 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,+ 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,+ 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,+ 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,+ 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,+ 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,+ 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,+ 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,+ 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,+ 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,+ 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,+ 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,+ 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,+ 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,+ 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,+ 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,+ 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,+ 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,+ 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,+ 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,+ 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,+ 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,+ 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,+ 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,+ 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,+ 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,+ 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,+ 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,+ 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,+ 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,+ 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,+ 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,+ 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,+ 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,+ 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,+ 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,+ 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,+ 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,+ 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,+ 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,+ 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,+ 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,+ 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,+ 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,+ 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,+ 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,+ 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,+ 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,+ 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,+ 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,+ 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,+ 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,+ 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,+ 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,+ 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,+ 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,+ 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,+ 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,+ 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,+ 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,+ 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,+ 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,+ 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,+ 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,+ 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,+ 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,+ 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,+ 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,+ 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,+ 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,+ 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,+ 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,+ 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,+ 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,+ 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,+ 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,+ 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,+ 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,+ 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,+ 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,+ 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,+ 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,+ 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,+ 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,+ 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,+ 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,+ 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,+ 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,+ 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,+ 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,+ 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,+ 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,+ 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,+ 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,+ 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,+ 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,+ 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,+ 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,+ 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,+ 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,+ 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,+ 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,+ 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,+ 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,+ 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,+ 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,+ 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,+ 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,+ 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,+ 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,+ 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,+ 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,+ 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,+ 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,+ 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,+ 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,+ 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,+ 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,+ 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,+ 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,+ 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,+ 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,+ 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,+ 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,+ 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,+ 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,+ 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256)+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)+ 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,+ 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,+ 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,+ 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,+ 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,+ 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,+ 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,+ 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,+ 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,+ 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,+ 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,+ 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,+ 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,+ 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,+ 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,+ 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,+ 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,+ 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,+ 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,+ 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,+ 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,+ 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,+ 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,+ 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,+ 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,+ 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,+ 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,+ 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,+ 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,+ 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,+ 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,+ 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,+ 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,+ 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,+ 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,+ 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,+ 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,+ 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,+ 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,+ 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,+ 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,+ 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,+ 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,+ 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,+ 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,+ 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,+ 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,+ 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,+ 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,+ 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,+ 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,+ 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,+ 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,+ 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,+ 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,+ 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,+ 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,+ 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,+ 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,+ 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,+ 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,+ 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,+ 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,+ 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,+GGML_TABLE_END()++#define NGRID_IQ1S 2048+#define IQ1S_DELTA 0.125f+#if defined(GGML_COMMON_IMPL_C)+GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)+ 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,+ 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,+ 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000,+ 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000,+ 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101,+ 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff,+ 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00,+ 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101,+ 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100,+ 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00,+ 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000,+ 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000,+ 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101,+ 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff,+ 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100,+ 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01,+ 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000,+ 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff,+ 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001,+ 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000,+ 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00,+ 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff,+ 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff,+ 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000,+ 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff,+ 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff,+ 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101,+ 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff,+ 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000,+ 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100,+ 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01,+ 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00,+ 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00,+ 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01,+ 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff,+ 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000,+ 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff,+ 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000,+ 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101,+ 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100,+ 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff,+ 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff,+ 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001,+ 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01,+ 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff,+ 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000,+ 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000,+ 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff,+ 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01,+ 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00,+ 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000,+ 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000,+ 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000,+ 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001,+ 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff,+ 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01,+ 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff,+ 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff,+ 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000,+ 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000,+ 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00,+ 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001,+ 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100,+ 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101,+ 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00,+ 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00,+ 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000,+ 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001,+ 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff,+ 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000,+ 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000,+ 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff,+ 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100,+ 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000,+ 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101,+ 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001,+ 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000,+ 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff,+ 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01,+ 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100,+ 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000,+ 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01,+ 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101,+ 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000,+ 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff,+ 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff,+ 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001,+ 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001,+ 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000,+ 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000,+ 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00,+ 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100,+ 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01,+ 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00,+ 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff,+ 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000,+ 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00,+ 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000,+ 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff,+ 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00,+ 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000,+ 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000,+ 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff,+ 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00,+ 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00,+ 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001,+ 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001,+ 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000,+ 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100,+ 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101,+ 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101,+ 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000,+ 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00,+ 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff,+ 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000,+ 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff,+ 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff,+ 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff,+ 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101,+ 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001,+ 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100,+ 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101,+ 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01,+ 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00,+ 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00,+ 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000,+ 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101,+ 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100,+ 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001,+ 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff,+ 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00,+ 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000,+ 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100,+ 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00,+ 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01,+ 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff,+ 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101,+ 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100,+ 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100,+ 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000,+ 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff,+ 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff,+ 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000,+ 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101,+ 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000,+ 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101,+ 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101,+ 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100,+ 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01,+ 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001,+ 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001,+ 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff,+ 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff,+ 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000,+ 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000,+ 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101,+ 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff,+ 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001,+ 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000,+ 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff,+ 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00,+ 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff,+ 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000,+ 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00,+ 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff,+ 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000,+ 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000,+ 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff,+ 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000,+ 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000,+ 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000,+ 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101,+ 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff,+ 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100,+ 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101,+ 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001,+ 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff,+ 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100,+ 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff,+ 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01,+ 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff,+ 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff,+ 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff,+ 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff,+ 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001,+ 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff,+ 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01,+ 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00,+ 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100,+ 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101,+ 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff,+ 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00,+ 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01,+ 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00,+ 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100,+ 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00,+ 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000,+ 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000,+ 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000,+ 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000,+ 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001,+ 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff,+ 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00,+ 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff,+ 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00,+ 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000,+ 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff,+ 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff,+ 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101,+ 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000,+ 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff,+ 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff,+ 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff,+ 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff,+ 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000,+ 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000,+ 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100,+ 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00,+ 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100,+ 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100,+ 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00,+ 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff,+ 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff,+ 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100,+ 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff,+ 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000,+ 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00,+ 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,+ 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00,+ 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100,+ 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff,+ 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff,+ 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001,+ 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00,+ 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101,+ 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001,+ 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000,+ 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100,+ 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000,+ 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001,+ 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000,+ 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff,+ 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101,+ 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,+ 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101,+ 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000,+ 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff,+ 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000,+ 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff,+ 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01,+ 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100,+ 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff,+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101,+ 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001,+ 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01,+ 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff,+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,+ 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff,+ 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00,+ 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff,+ 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff,+ 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff,+ 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001,+ 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff,+ 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff,+ 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001,+ 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff,+ 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff,+ 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000,+ 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00,+ 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100,+ 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01,+ 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff,+ 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff,+ 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000,+ 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101,+ 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01,+ 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100,+ 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100,+ 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00,+ 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101,+ 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001,+ 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01,+ 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100,+ 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff,+ 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01,+ 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff,+ 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000,+ 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001,+ 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01,+ 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff,+ 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff,+ 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00,+ 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff,+ 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100,+ 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000,+ 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001,+ 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100,+ 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff,+ 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff,+ 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01,+ 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101,+ 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001,+ 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff,+ 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101,+ 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100,+ 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00,+ 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01,+ 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001,+ 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00,+ 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000,+ 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101,+ 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001,+ 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100,+ 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000,+ 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000,+ 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00,+ 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101,+ 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff,+ 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101,+ 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001,+ 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01,+ 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100,+ 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000,+ 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00,+ 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff,+ 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001,+ 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001,+ 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff,+ 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00,+ 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100,+ 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00,+ 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101,+ 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff,+ 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff,+ 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff,+ 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00,+ 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff,+ 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff,+ 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101,+ 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100,+ 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101,+ 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00,+ 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00,+ 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff,+ 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff,+ 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001,+ 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000,+ 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff,+ 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff,+ 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff,+ 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff,+ 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001,+ 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100,+ 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101,+ 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001,+ 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001,+ 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101,+ 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101,+ 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff,+ 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff,+ 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000,+ 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101,+ 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001,+ 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff,+ 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000,+ 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff,+ 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100,+ 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000,+ 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101,+ 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff,+ 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100,+ 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff,+ 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01,+ 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100,+ 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000,+ 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100,+ 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100,+ 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001,+ 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000,+ 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000,+ 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000,+ 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00,+ 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff,+ 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001,+ 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000,+ 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101,+ 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101,+ 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000,+ 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff,+ 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000,+ 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101,+ 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff,+ 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff,+ 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000,+ 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101,+ 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff,+ 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff,+ 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01,+ 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff,+ 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000,+ 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101,+ 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff,+ 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff,+ 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff,+ 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01,+ 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00,+ 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000,+ 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000,+ 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101,+ 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001,+ 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff,+ 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000,+ 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff,+ 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000,+ 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000,+ 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000,+ 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000,+ 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001,+ 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff,+ 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001,+ 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000,+ 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000,+ 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00,+ 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000,+ 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101,+ 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001,+ 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00,+ 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001,+ 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff,+ 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000,+ 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00,+ 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100,+ 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101,+ 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001,+ 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01,+ 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff,+ 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff,+ 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00,+ 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01,+ 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100,+ 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000,+ 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff,+ 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff,+ 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff,+ 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000,+ 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff,+ 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101,+ 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff,+ 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001,+ 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001,+ 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001,+ 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000,+ 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000,+ 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff,+ 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101,+ 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001,+ 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff,+ 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000,+ 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000,+ 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff,+ 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101,+ 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000,+ 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100,+ 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00,+ 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000,+ 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff,+ 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01,+ 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00,+ 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff,+ 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000,+ 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101,+ 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff,+ 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001,+ 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff,+ 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000,+ 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100,+ 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff,+ 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101,+ 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100,+ 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff,+ 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01,+ 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000,+ 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff,+ 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00,+ 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100,+ 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff,+ 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001,+ 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00,+ 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100,+ 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff,+ 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01,+ 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00,+ 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000,+ 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01,+ 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff,+ 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000,+ 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff,+ 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff,+ 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff,+ 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001,+ 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff,+ 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01,+ 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff,+ 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00,+ 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00,+ 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001,+ 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101,+ 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101,+ 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff,+ 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000,+ 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101,+GGML_TABLE_END()+#else+GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S)+ 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,+ 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,+ 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,+ 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,+ 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,+ 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,+ 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,+ 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,+ 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,+ 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,+ 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,+ 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,+ 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,+ 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,+ 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,+ 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,+ 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,+ 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,+ 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,+ 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,+ 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,+ 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,+ 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,+ 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,+ 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,+ 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,+ 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,+ 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,+ 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,+ 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,+ 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,+ 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,+ 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,+ 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,+ 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,+ 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,+ 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,+ 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,+ 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,+ 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,+ 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,+ 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,+ 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,+ 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,+ 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,+ 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,+ 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,+ 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,+ 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,+ 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,+ 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,+ 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,+ 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,+ 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,+ 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,+ 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,+ 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,+ 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,+ 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,+ 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,+ 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,+ 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,+ 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,+ 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,+ 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,+ 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,+ 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,+ 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,+ 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,+ 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,+ 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,+ 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,+ 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,+ 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,+ 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,+ 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,+ 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,+ 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,+ 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,+ 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,+ 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,+ 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,+ 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,+ 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,+ 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,+ 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,+ 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,+ 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,+ 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,+ 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,+ 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,+ 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,+ 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,+ 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,+ 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,+ 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,+ 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,+ 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,+ 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,+ 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,+ 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,+ 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,+ 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,+ 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,+ 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,+ 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,+ 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,+ 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,+ 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,+ 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,+ 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,+ 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,+ 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,+ 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,+ 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,+ 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,+ 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,+ 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,+ 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,+ 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,+ 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,+ 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,+ 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,+ 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,+ 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,+ 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,+ 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,+ 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,+ 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,+ 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,+ 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,+ 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,+ 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,+ 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,+ 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,+ 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,+ 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,+ 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,+ 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,+ 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,+ 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,+ 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,+ 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,+ 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,+ 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,+ 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,+ 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,+ 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,+ 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,+ 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,+ 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,+ 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,+ 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,+ 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,+ 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,+ 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,+ 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,+ 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,+ 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,+ 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,+ 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,+ 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,+ 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,+ 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,+ 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,+ 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,+ 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,+ 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,+ 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,+ 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,+ 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,+ 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,+ 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,+ 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,+ 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,+ 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,+ 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,+ 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,+ 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,+ 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,+ 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,+ 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,+ 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,+ 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,+ 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,+ 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,+ 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,+ 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,+ 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,+ 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,+ 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,+ 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,+ 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,+ 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,+ 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,+ 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,+ 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,+ 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,+ 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,+ 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,+ 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,+ 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,+ 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,+ 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,+ 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,+ 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,+ 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,+ 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,+ 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,+ 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,+ 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,+ 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,+ 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,+ 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,+ 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,+ 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,+ 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,+ 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,+ 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,+ 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,+ 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,+ 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,+ 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,+ 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,+ 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,+ 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,+ 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,+ 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,+ 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,+ 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,+ 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,+ 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,+ 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,+ 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,+ 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,+ 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,+ 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,+ 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,+ 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,+ 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,+ 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,+ 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,+ 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,+ 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,+ 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,+ 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,+ 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,+ 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,+ 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,+ 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,+ 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,+ 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,+ 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,+ 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,+ 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,+ 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,+GGML_TABLE_END()+#endif++#endif // GGML_COMMON_IMPL+#endif // GGML_COMMON_IMPLdiff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 9c710fdd..44634c62 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -2,6 +2,7 @@// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#include <algorithm>+#include <array>#include <assert.h>#include <atomic>#include <cinttypes>@@ -9,12 +10,22 @@#include <cstdint>#include <float.h>#include <limits>+#include <map>+#include <memory>+#include <mutex>#include <stdint.h>#include <stdio.h>#include <string>#include <vector>-#include <map>-#include <array>++#if defined(GGML_USE_HIPBLAS)+#define GGML_COMMON_DECL_HIP+#define GGML_COMMON_IMPL_HIP+#else+#define GGML_COMMON_DECL_CUDA+#define GGML_COMMON_IMPL_CUDA+#endif+#include "ggml-common.h"#ifdef __i386__#error "you need to use a 64-bit compiler for llamafile"@@ -45,6 +56,7 @@#define cublasOperation_t tinyblasOperation_t#define cublasComputeType_t tinyblasComputeType_t#define cublasCreate tinyblasCreate+#define cublasDestroy tinyblasDestroy#define cublasGemmEx tinyblasGemmEx#define cublasGemmBatchedEx tinyblasGemmBatchedEx#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx@@ -65,6 +77,7 @@#define cudaEventCreateWithFlags hipEventCreateWithFlags#define cudaEventDisableTiming hipEventDisableTiming#define cudaEventRecord hipEventRecord+#define cudaEventSynchronize hipEventSynchronize#define cudaEvent_t hipEvent_t#define cudaEventDestroy hipEventDestroy#define cudaFree hipFree@@ -74,6 +87,11 @@#define cudaGetDeviceProperties hipGetDeviceProperties#define cudaGetErrorString hipGetErrorString#define cudaGetLastError hipGetLastError+#define cudaHostRegister hipHostRegister+#define cudaHostRegisterPortable hipHostRegisterPortable+#define cudaHostRegisterReadOnly hipHostRegisterReadOnly+#define cudaHostUnregister hipHostUnregister+#define cudaLaunchHostFunc hipLaunchHostFunc#ifdef GGML_HIP_UMA#define cudaMalloc hipMallocManaged#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size)@@ -95,8 +113,10 @@#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize#define cudaSetDevice hipSetDevice#define cudaStreamCreateWithFlags hipStreamCreateWithFlags+#define cudaStreamDestroy hipStreamDestroy#define cudaStreamFireAndForget hipStreamFireAndForget#define cudaStreamNonBlocking hipStreamNonBlocking+#define cudaStreamPerThread hipStreamPerThread#define cudaStreamSynchronize hipStreamSynchronize#define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags)#define cudaStream_t hipStream_t@@ -127,6 +147,7 @@#define cublasSgemm tinyblasSgemm#define cublasGemmEx tinyblasGemmEx#define cublasCreate tinyblasCreate+#define cublasDestroy tinyblasDestroy#define cublasSetStream tinyblasSetStream#define cublasGemmBatchedEx tinyblasGemmBatchedEx#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx@@ -144,7 +165,7 @@#endif // __HIP_PLATFORM_AMD__#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F-#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_R_32F#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT#define CUBLAS_OP_N HIPBLAS_OP_N@@ -156,6 +177,7 @@#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)#define cublasComputeType_t hipblasDatatype_t //deprecated, new hipblasComputeType_t not in 5.6#define cublasCreate hipblasCreate+#define cublasDestroy hipblasDestroy#define cublasGemmEx hipblasGemmEx#define cublasGemmBatchedEx hipblasGemmBatchedEx#define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx@@ -164,7 +186,6 @@#define cublasSetStream hipblasSetStream#define cublasSgemm hipblasSgemm#define cublasStatus_t hipblasStatus_t-#define cublasGetStatusString hipblasStatusToString#define cudaDataType_t hipblasDatatype_t //deprecated, new hipblasDatatype not in 5.6#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess@@ -177,6 +198,7 @@#define cudaEventCreateWithFlags hipEventCreateWithFlags#define cudaEventDisableTiming hipEventDisableTiming#define cudaEventRecord hipEventRecord+#define cudaEventSynchronize hipEventSynchronize#define cudaEvent_t hipEvent_t#define cudaEventDestroy hipEventDestroy#define cudaFree hipFree@@ -186,6 +208,11 @@#define cudaGetDeviceProperties hipGetDeviceProperties#define cudaGetErrorString hipGetErrorString#define cudaGetLastError hipGetLastError+#define cudaHostRegister hipHostRegister+#define cudaHostRegisterPortable hipHostRegisterPortable+#define cudaHostRegisterReadOnly hipHostRegisterReadOnly+#define cudaHostUnregister hipHostUnregister+#define cudaLaunchHostFunc hipLaunchHostFunc#ifdef GGML_HIP_UMA#define cudaMalloc hipMallocManaged#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size)@@ -207,8 +234,10 @@#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize#define cudaSetDevice hipSetDevice#define cudaStreamCreateWithFlags hipStreamCreateWithFlags+#define cudaStreamDestroy hipStreamDestroy#define cudaStreamFireAndForget hipStreamFireAndForget#define cudaStreamNonBlocking hipStreamNonBlocking+#define cudaStreamPerThread hipStreamPerThread#define cudaStreamSynchronize hipStreamSynchronize#define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags)#define cudaStream_t hipStream_t@@ -223,8 +252,6 @@#define CUBLAS_STATUS_EXECUTION_FAILED HIPBLAS_STATUS_EXECUTION_FAILED#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED-#define CUDART_VERSION 12000-#else#include <cuda_runtime.h>#include <cuda.h>@@ -273,7 +300,7 @@ static const struct ggml_backend_api *g_backend;#define ggml_backend_buffer_init g_backend->ggml_backend_buffer_init#define ggml_backend_cpu_buffer_from_ptr g_backend->ggml_backend_cpu_buffer_from_ptr-GGML_CALL void ggml_cuda_link(const struct ggml_backend_api *backend_api) {+GGML_CALL bool ggml_cuda_link(const struct ggml_backend_api *backend_api) {g_backend = backend_api;if (!FLAG_log_disable) {fprintf(stderr, "%s: welcome to " GGML_CUDA_NAME " SDK with "@@ -284,6 +311,8 @@ GGML_CALL void ggml_cuda_link(const struct ggml_backend_api *backend_api) {#endif"\n", __func__);}+ int device_count;+ return cudaGetDeviceCount(&device_count) == cudaSuccess && device_count > 0;}#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)@@ -296,8 +325,6 @@ GGML_CALL void ggml_cuda_link(const struct ggml_backend_api *backend_api) {#define CC_RDNA2 (CC_OFFSET_AMD + 1030)#define CC_RDNA3 (CC_OFFSET_AMD + 1100)-#define GGML_CUDA_MAX_NODES 8192-// define this if you want to always fallback to MMQ kernels and not use cuBLAS for matrix multiplication// on modern hardware, using cuBLAS is recommended as it utilizes F16 tensor cores which are very performant// for large computational tasks. the drawback is that this requires some extra amount of VRAM:@@ -315,88 +342,8 @@ GGML_CALL void ggml_cuda_link(const struct ggml_backend_api *backend_api) {#define MMVQ_MAX_BATCH_SIZE 8 // max batch size to use MMVQ kernels#define MMQ_MAX_BATCH_SIZE 32 // max batch size to use MMQ kernels when tensor cores are available-#if defined(GGML_USE_HIPBLAS)-#define __CUDA_ARCH__ 1300--#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \- defined(__gfx1150__) || defined(__gfx1151__)-#define RDNA3-#endif--#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \- defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)-#define RDNA2-#endif--#ifndef __has_builtin- #define __has_builtin(x) 0-#endif--typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));-typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));-static __device__ __forceinline__ int __vsubss4(const int a, const int b) {- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);-#if __has_builtin(__builtin_elementwise_sub_sat)- const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);- return reinterpret_cast<const int &>(c);-#else- int8x4_t c;- int16_t tmp;-#pragma unroll- for (int i = 0; i < 4; i++) {- tmp = va[i] - vb[i];- if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();- if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();- c[i] = tmp;- }- return reinterpret_cast<int &>(c);-#endif // __has_builtin(__builtin_elementwise_sub_sat)-}--static __device__ __forceinline__ int __vsub4(const int a, const int b) {- return __vsubss4(a, b);-}--static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {- const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);- const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);- unsigned int c;- uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);-#pragma unroll- for (int i = 0; i < 4; ++i) {- vc[i] = va[i] == vb[i] ? 0xff : 0x00;- }- return c;-}+#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses-static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {-#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx1030__)- c = __builtin_amdgcn_sdot4(a, b, c, false);-#elif defined(RDNA3)- c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);-#elif defined(__gfx1010__) || defined(__gfx900__)- int tmp1;- int tmp2;- asm("\n \- v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \- v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \- v_add3_u32 %0, %1, %2, %0 \n \- v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \- v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \- v_add3_u32 %0, %1, %2, %0 \n \- "- : "+v"(c), "=&v"(tmp1), "=&v"(tmp2)- : "v"(a), "v"(b)- );-#else- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);- c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];-#endif- return c;-}-#endif // defined(GGML_USE_HIPBLAS)#if defined(_MSC_VER)#pragma warning(disable: 4244 4267) // possible loss of data@@ -464,4327 +411,3759 @@ static const char * cu_get_error_str(CUresult err) {#define GGML_CUDA_ASSUME(x)#endif // CUDART_VERSION >= 11100-#ifdef GGML_CUDA_F16-typedef half dfloat; // dequantize float-typedef half2 dfloat2;-#else-typedef float dfloat; // dequantize float-typedef float2 dfloat2;-#endif //GGML_CUDA_F16--static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {- const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment-- int x32 = 0;- x32 |= x16[0] << 0;- x32 |= x16[1] << 16;- return x32;-}+#define GGML_CUDA_MAX_STREAMS 8-static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {- const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment+struct ggml_tensor_extra_gpu {+ void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors+ cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs+};- int x32 = 0;- x32 |= x16[0] << 0;- x32 |= x16[1] << 16;+// this is faster on Windows+// probably because the Windows CUDA libraries forget to make this check before invoking the drivers+static void ggml_cuda_set_device(const int device) {+ int current_device;+ CUDA_CHECK(cudaGetDevice(¤t_device));- return x32;-}+ if (device == current_device) {+ return;+ }-static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {- return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+ CUDA_CHECK(cudaSetDevice(device));}-static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {- return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+static int ggml_cuda_get_device() {+ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ return id;}-template<typename T>-using to_t_cuda_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int k, cudaStream_t stream);-typedef to_t_cuda_t<float> to_fp32_cuda_t;-typedef to_t_cuda_t<half> to_fp16_cuda_t;--typedef void (*dequantize_kernel_t)(const void * vx, const int ib, const int iqs, dfloat2 & v);-typedef void (*dot_kernel_k_t)(const void * __restrict__ vx, const int ib, const int iqs, const float * __restrict__ y, float & v);-typedef void (*cpy_kernel_t)(const char * cx, char * cdst);-typedef void (*ggml_cuda_func_t)(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);-typedef void (*ggml_cuda_op_mul_mat_t)(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream);-typedef void (*ggml_cuda_op_flatten_t)(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream);--// QK = number of values after dequantization-// QR = QK / number of values before dequantization-// QI = number of 32 bit integers before dequantization--#define QK4_0 32-#define QR4_0 2-#define QI4_0 (QK4_0 / (4 * QR4_0))-typedef struct {- half d; // delta- uint8_t qs[QK4_0 / 2]; // nibbles / quants-} block_q4_0;-static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 block size/padding");--#define QK4_1 32-#define QR4_1 2-#define QI4_1 (QK4_1 / (4 * QR4_1))-typedef struct {- half2 dm; // dm.x = delta, dm.y = min- uint8_t qs[QK4_1 / 2]; // nibbles / quants-} block_q4_1;-static_assert(sizeof(block_q4_1) == sizeof(ggml_fp16_t) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding");--#define QK5_0 32-#define QR5_0 2-#define QI5_0 (QK5_0 / (4 * QR5_0))-typedef struct {- half d; // delta- uint8_t qh[4]; // 5-th bit of quants- uint8_t qs[QK5_0 / 2]; // nibbles / quants-} block_q5_0;-static_assert(sizeof(block_q5_0) == sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");--#define QK5_1 32-#define QR5_1 2-#define QI5_1 (QK5_1 / (4 * QR5_1))-typedef struct {- half2 dm; // dm.x = delta, dm.y = min- uint8_t qh[4]; // 5-th bit of quants- uint8_t qs[QK5_1 / 2]; // nibbles / quants-} block_q5_1;-static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");--#define QK8_0 32-#define QR8_0 1-#define QI8_0 (QK8_0 / (4 * QR8_0))-typedef struct {- half d; // delta- int8_t qs[QK8_0]; // quants-} block_q8_0;-static_assert(sizeof(block_q8_0) == sizeof(ggml_fp16_t) + QK8_0, "wrong q8_0 block size/padding");--#define QK8_1 32-#define QR8_1 1-#define QI8_1 (QK8_1 / (4 * QR8_1))-typedef struct {- half2 ds; // ds.x = delta, ds.y = sum- int8_t qs[QK8_0]; // quants-} block_q8_1;-static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_fp16_t) + QK8_0, "wrong q8_1 block size/padding");+struct ggml_cuda_device_info {+ int device_count;-typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs);-typedef void (*allocate_tiles_cuda_t)(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc);-typedef void (*load_tiles_cuda_t)(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row);-typedef float (*vec_dot_q_mul_mat_cuda_t)(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k);+ struct cuda_device_info {+ int cc; // compute capability+ size_t smpb; // max. shared memory per block+ bool vmm; // virtual memory support+ size_t vmm_granularity; // granularity of virtual memory+ size_t total_vram;+ };-//================================= k-quants+ cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {};-#ifdef GGML_QKK_64-#define QK_K 64-#define K_SCALE_SIZE 4-#else-#define QK_K 256-#define K_SCALE_SIZE 12-#endif+ std::array<float, GGML_CUDA_MAX_DEVICES> default_tensor_split = {};+};-#define QR2_K 4-#define QI2_K (QK_K / (4*QR2_K))-typedef struct {- uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits- uint8_t qs[QK_K/4]; // quants- half2 dm; // super-block scale for quantized scales/mins-} block_q2_K;-static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");--#define QR3_K 4-#define QI3_K (QK_K / (4*QR3_K))-typedef struct {- uint8_t hmask[QK_K/8]; // quants - high bit- uint8_t qs[QK_K/4]; // quants - low 2 bits-#ifdef GGML_QKK_64- uint8_t scales[2]; // scales, quantized with 8 bits-#else- uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits-#endif- half d; // super-block scale-} block_q3_K;-//static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + K_SCALE_SIZE, "wrong q3_K block size/padding");--#define QR4_K 2-#define QI4_K (QK_K / (4*QR4_K))-#ifdef GGML_QKK_64-typedef struct {- half dm[2]; // super-block scales/mins- uint8_t scales[2]; // 4-bit block scales/mins- uint8_t qs[QK_K/2]; // 4--bit quants-} block_q4_K;-static_assert(sizeof(block_q4_K) == sizeof(half2) + QK_K/2 + 2, "wrong q4_K block size/padding");-#else-typedef struct {- half2 dm; // super-block scale for quantized scales/mins- uint8_t scales[3*QK_K/64]; // scales, quantized with 6 bits- uint8_t qs[QK_K/2]; // 4--bit quants-} block_q4_K;-static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + 3*QK_K/64 + QK_K/2, "wrong q4_K block size/padding");+static ggml_cuda_device_info ggml_cuda_init() {+#ifdef __HIP_PLATFORM_AMD__+ // Workaround for a rocBLAS bug when using multiple graphics cards:+ // https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1346+#ifndef GGML_USE_TINYBLAS+ rocblas_initialize();+ CUDA_CHECK(cudaDeviceSynchronize());#endif--#define QR5_K 2-#define QI5_K (QK_K / (4*QR5_K))-#ifdef GGML_QKK_64-typedef struct {- half d; // super-block scale- int8_t scales[QK_K/16]; // block scales- uint8_t qh[QK_K/8]; // quants, high bit- uint8_t qs[QK_K/2]; // quants, low 4 bits-} block_q5_K;-static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding");-#else-typedef struct {- half2 dm; // super-block scale for quantized scales/mins- uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits- uint8_t qh[QK_K/8]; // quants, high bit- uint8_t qs[QK_K/2]; // quants, low 4 bits-} block_q5_K;-static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");#endif-#define QR6_K 2-#define QI6_K (QK_K / (4*QR6_K))-typedef struct {- uint8_t ql[QK_K/2]; // quants, lower 4 bits- uint8_t qh[QK_K/4]; // quants, upper 2 bits- int8_t scales[QK_K/16]; // scales- half d; // delta-} block_q6_K;-static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_K block size/padding");--#define QR2_XXS 8-#define QI2_XXS (QK_K / (4*QR2_XXS))-typedef struct {- half d;- uint16_t qs[QK_K/8];-} block_iq2_xxs;-static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");--#define QR2_XS 8-#define QI2_XS (QK_K / (4*QR2_XS))-typedef struct {- half d;- uint16_t qs[QK_K/8];- uint8_t scales[QK_K/32];-} block_iq2_xs;-static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");--// 2.5625 bpw quants-#define QR2_S 8-#define QI2_S (QK_K / (4*QR2_S))-typedef struct {- half d;- uint8_t qs[QK_K/4];- uint8_t qh[QK_K/32];- uint8_t scales[QK_K/32];-} block_iq2_s;-static_assert(sizeof(block_iq2_s) == sizeof(ggml_fp16_t) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding");--#define QR3_XXS 8-#define QI3_XXS (QK_K / (4*QR3_XXS))-typedef struct {- half d;- uint8_t qs[3*(QK_K/8)];-} block_iq3_xxs;-static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");--#define QR3_XS 8-#define QI3_XS (QK_K / (4*QR3_XS))-typedef struct {- half d;- uint8_t qs[QK_K/4];- uint8_t qh[QK_K/32];- uint8_t signs[QK_K/8];- uint8_t scales[QK_K/64];-} block_iq3_s;-static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 27*(QK_K/64), "wrong iq3_s block size/padding");--#define QR1_S 8-#define QI1_S (QK_K / (4*QR1_S))-typedef struct {- half d;- uint8_t qs[QK_K/8];- uint8_t scales[QK_K/16];-} block_iq1_s;-static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");--#define QK4_NL 32-#define QR4_NL 2-#define QI4_NL (QK4_NL / (4*QR4_NL))-typedef struct {- half d;- uint8_t qs[QK4_NL/2];-} block_iq4_nl;-static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding");--// QR4_XS = 8 is very slightly faster than QR4_XS = 4-#define QR4_XS 8-#define QI4_XS (QK_K / (4*QR4_XS))-typedef struct {- half d;- uint16_t scales_h;- uint8_t scales_l[QK_K/64];- uint8_t qs[QK_K/2];-} block_iq4_xs;-static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");---#define WARP_SIZE 32-#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses+ ggml_cuda_device_info info = {};-#define CUDA_GELU_BLOCK_SIZE 256-#define CUDA_SILU_BLOCK_SIZE 256-#define CUDA_TANH_BLOCK_SIZE 256-#define CUDA_RELU_BLOCK_SIZE 256-#define CUDA_HARDSIGMOID_BLOCK_SIZE 256-#define CUDA_HARDSWISH_BLOCK_SIZE 256-#define CUDA_SQR_BLOCK_SIZE 256-#define CUDA_CPY_BLOCK_SIZE 32-#define CUDA_SCALE_BLOCK_SIZE 256-#define CUDA_CLAMP_BLOCK_SIZE 256-#define CUDA_ROPE_BLOCK_SIZE 256-#define CUDA_SOFT_MAX_BLOCK_SIZE 1024-#define CUDA_ALIBI_BLOCK_SIZE 32-#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32-#define CUDA_QUANTIZE_BLOCK_SIZE 256-#define CUDA_DEQUANTIZE_BLOCK_SIZE 256-#define CUDA_GET_ROWS_BLOCK_SIZE 256-#define CUDA_UPSCALE_BLOCK_SIZE 256-#define CUDA_CONCAT_BLOCK_SIZE 256-#define CUDA_PAD_BLOCK_SIZE 256-#define CUDA_ACC_BLOCK_SIZE 256-#define CUDA_IM2COL_BLOCK_SIZE 256-#define CUDA_POOL2D_BLOCK_SIZE 256+ cudaError_t err = cudaGetDeviceCount(&info.device_count);+ if (err != cudaSuccess) {+ fprintf(stderr, "%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err));+ return info;+ }-#define CUDA_Q8_0_NE_ALIGN 2048+ GGML_ASSERT(info.device_count <= GGML_CUDA_MAX_DEVICES);-// dmmv = dequantize_mul_mat_vec-#ifndef GGML_CUDA_DMMV_X-#define GGML_CUDA_DMMV_X 32-#endif-#ifndef GGML_CUDA_MMV_Y-#define GGML_CUDA_MMV_Y 1+ int64_t total_vram = 0;+if (!FLAG_log_disable)+#if defined(GGML_CUDA_FORCE_MMQ)+ fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__);+#else+ fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: no\n", __func__);#endif--#ifndef K_QUANTS_PER_ITERATION-#define K_QUANTS_PER_ITERATION 2+if (!FLAG_log_disable)+#if defined(CUDA_USE_TENSOR_CORES)+ fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: yes\n", __func__);#else-static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUANTS_PER_ITERATION must be 1 or 2");+ fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: no\n", __func__);#endif+ fprintf(stderr, "%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count);+ for (int id = 0; id < info.device_count; ++id) {+ int device_vmm = 0;-#ifndef GGML_CUDA_PEER_MAX_BATCH_SIZE-#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128-#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE+#if !defined(GGML_USE_HIPBLAS)+ CUdevice device;+ CU_CHECK(cuDeviceGet(&device, id));+ CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device));-#define MUL_MAT_SRC1_COL_STRIDE 128+ if (device_vmm) {+ CUmemAllocationProp alloc_prop = {};+ alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;+ alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;+ alloc_prop.location.id = id;+ CU_CHECK(cuMemGetAllocationGranularity(&info.devices[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));+ }+#endif // !defined(GGML_USE_HIPBLAS)+ info.devices[id].vmm = !!device_vmm;-#define MAX_STREAMS 8-static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { { nullptr } };+ cudaDeviceProp prop;+ CUDA_CHECK(cudaGetDeviceProperties(&prop, id));+ if (!FLAG_log_disable)+ fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");-struct ggml_tensor_extra_gpu {- void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors- cudaEvent_t events[GGML_CUDA_MAX_DEVICES][MAX_STREAMS]; // events for synchronizing multiple GPUs-};+ info.default_tensor_split[id] = total_vram;+ total_vram += prop.totalGlobalMem;-// this is faster on Windows-// probably because the Windows CUDA libraries forget to make this check before invoking the drivers-static void ggml_cuda_set_device(const int device) {- int current_device;- CUDA_CHECK(cudaGetDevice(¤t_device));+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ info.devices[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;+#else+ info.devices[id].cc = 100*prop.major + 10*prop.minor;+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ info.devices[id].smpb = prop.sharedMemPerBlock;+ }- if (device == current_device) {- return;+ for (int id = 0; id < info.device_count; ++id) {+ info.default_tensor_split[id] /= total_vram;}- CUDA_CHECK(cudaSetDevice(device));+ // configure logging to stdout+ // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));++ return info;}-static int g_device_count = -1;-static int g_main_device = 0;-static std::array<float, GGML_CUDA_MAX_DEVICES> g_default_tensor_split = {};+static const ggml_cuda_device_info & get_cuda_global_info() {+ static ggml_cuda_device_info info = ggml_cuda_init();+ return info;+}-struct cuda_device_capabilities {- int cc; // compute capability- size_t smpb; // max. shared memory per block- bool vmm; // virtual memory support- size_t vmm_granularity; // granularity of virtual memory-};+// #define DEBUG_CUDA_MALLOC-static cuda_device_capabilities g_device_caps[GGML_CUDA_MAX_DEVICES] = { {0, 0, false, 0} };+// buffer pool for cuda (legacy)+struct ggml_cuda_pool {+ virtual ~ggml_cuda_pool() = default;-static cublasHandle_t g_cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};+ virtual void * alloc(size_t size, size_t * actual_size) = 0;+ virtual void free(void * ptr, size_t size) = 0;+};-[[noreturn]]-static __device__ void no_device_code(- const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {+struct ggml_cuda_pool_leg : public ggml_cuda_pool {+ static const int MAX_BUFFERS = 256;-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n",- file_name, line, function_name, arch);- (void) arch_list;-#else- printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n",- file_name, line, function_name, arch, arch_list);-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- __trap();+ int device;+ struct ggml_cuda_buffer {+ void * ptr = nullptr;+ size_t size = 0;+ };- (void) no_device_code; // suppress unused function warning-}+ ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {};+ size_t pool_size = 0;-#ifdef __CUDA_ARCH__-#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))-#else-#define NO_DEVICE_CODE GGML_ASSERT(false && "NO_DEVICE_CODE not valid in host code.")-#endif // __CUDA_ARCH__--static __device__ __forceinline__ float warp_reduce_sum(float x) {-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- x += __shfl_xor_sync(0xffffffff, x, mask, 32);+ explicit ggml_cuda_pool_leg(int device) :+ device(device) {}- return x;-}-static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- a.x += __shfl_xor_sync(0xffffffff, a.x, mask, 32);- a.y += __shfl_xor_sync(0xffffffff, a.y, mask, 32);+ ~ggml_cuda_pool_leg() {+ ggml_cuda_set_device(device);+ for (int i = 0; i < MAX_BUFFERS; ++i) {+ ggml_cuda_buffer & b = buffer_pool[i];+ if (b.ptr != nullptr) {+ CUDA_CHECK(cudaFree(b.ptr));+ pool_size -= b.size;+ }+ }+ GGML_ASSERT(pool_size == 0);}- return a;-}--#ifdef GGML_CUDA_F16-static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));- }- return a;-#else- (void) a;- NO_DEVICE_CODE;-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL-}-#endif // GGML_CUDA_F16-static __device__ __forceinline__ float warp_reduce_max(float x) {-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, mask, 32));+ void * alloc(size_t size, size_t * actual_size) override {+#ifdef DEBUG_CUDA_MALLOC+ int nnz = 0;+ size_t max_size = 0;+#endif+ size_t best_diff = 1ull << 36;+ int ibest = -1;+ for (int i = 0; i < MAX_BUFFERS; ++i) {+ ggml_cuda_buffer& b = buffer_pool[i];+ if (b.ptr != nullptr) {+#ifdef DEBUG_CUDA_MALLOC+ ++nnz;+ if (b.size > max_size) max_size = b.size;+#endif+ if (b.size >= size) {+ size_t diff = b.size - size;+ if (diff < best_diff) {+ best_diff = diff;+ ibest = i;+ if (!best_diff) {+ void * ptr = b.ptr;+ *actual_size = b.size;+ b.ptr = nullptr;+ b.size = 0;+ return ptr;+ }+ }+ }+ }+ }+ if (ibest >= 0) {+ ggml_cuda_buffer& b = buffer_pool[ibest];+ void * ptr = b.ptr;+ *actual_size = b.size;+ b.ptr = nullptr;+ b.size = 0;+ return ptr;+ }+ void * ptr;+ size_t look_ahead_size = (size_t) (1.05 * size);+ look_ahead_size = 256 * ((look_ahead_size + 255)/256);+ ggml_cuda_set_device(device);+ CUDA_CHECK(cudaMalloc((void **) &ptr, look_ahead_size));+ *actual_size = look_ahead_size;+ pool_size += look_ahead_size;+#ifdef DEBUG_CUDA_MALLOC+ fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz,+ (uint32_t)(max_size/1024/1024), (uint32_t)(pool_size/1024/1024), (uint32_t)(size/1024/1024));+#endif+ return ptr;}- return x;-}-//static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {-//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX-//#pragma unroll-// for (int mask = 16; mask > 0; mask >>= 1) {-// x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));-// }-// return x;-//#else-// (void) x;-// NO_DEVICE_CODE;-//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX-//}+ void free(void * ptr, size_t size) override {+ for (int i = 0; i < MAX_BUFFERS; ++i) {+ ggml_cuda_buffer& b = buffer_pool[i];+ if (b.ptr == nullptr) {+ b.ptr = ptr;+ b.size = size;+ return;+ }+ }+ fprintf(stderr, "WARNING: cuda buffer pool full, increase MAX_CUDA_BUFFERS\n");+ ggml_cuda_set_device(device);+ CUDA_CHECK(cudaFree(ptr));+ pool_size -= size;+ }+};-static __device__ __forceinline__ float op_repeat(const float a, const float b) {- return b;- GGML_UNUSED(a);-}+// pool with virtual memory+#if !defined(GGML_USE_HIPBLAS)+struct ggml_cuda_pool_vmm : public ggml_cuda_pool {+ static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB-static __device__ __forceinline__ float op_add(const float a, const float b) {- return a + b;-}+ int device;+ CUdeviceptr pool_addr = 0;+ size_t pool_used = 0;+ size_t pool_size = 0;+ size_t granularity;-static __device__ __forceinline__ float op_mul(const float a, const float b) {- return a * b;-}+ explicit ggml_cuda_pool_vmm(int device) :+ device(device),+ granularity(get_cuda_global_info().devices[device].vmm_granularity) {+ }-static __device__ __forceinline__ float op_div(const float a, const float b) {- return a / b;-}+ ~ggml_cuda_pool_vmm() {+ if (pool_addr != 0) {+ CU_CHECK(cuMemUnmap(pool_addr, pool_size));+ CU_CHECK(cuMemAddressFree(pool_addr, CUDA_POOL_VMM_MAX_SIZE));+ }+ }-template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>-static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,- int ne0, int ne1, int ne2, int ne3,- int ne10, int ne11, int ne12, int ne13,- /*int s0, */ int s1, int s2, int s3,- /*int s10,*/ int s11, int s12, int s13) {- const int i0s = blockDim.x*blockIdx.x + threadIdx.x;- const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);- const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;- const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;+ void * alloc(size_t size, size_t * actual_size) override {+ // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types+ const size_t alignment = 128;+ size = alignment * ((size + alignment - 1) / alignment);- if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {- return;- }+ size_t avail = pool_size - pool_used;- const int i11 = i1 % ne11;- const int i12 = i2 % ne12;- const int i13 = i3 % ne13;+ if (size > avail) {+ // round up to the next multiple of the granularity+ size_t reserve_size = size - avail;+ reserve_size = granularity * ((reserve_size + granularity - 1) / granularity);- const size_t i_src0 = i3*s3 + i2*s2 + i1*s1;- const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;- const size_t i_dst = i_src0;+ GGML_ASSERT(pool_size + reserve_size <= CUDA_POOL_VMM_MAX_SIZE);- const src0_t * src0_row = src0 + i_src0;- const src1_t * src1_row = src1 + i_src1;- dst_t * dst_row = dst + i_dst;+ // allocate more physical memory+ CUmemAllocationProp prop = {};+ prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;+ prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;+ prop.location.id = device;+ CUmemGenericAllocationHandle handle;+ CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0));- for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {- const int i10 = i0 % ne10;- dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);- }-}+ // reserve virtual address space (if not already reserved)+ if (pool_addr == 0) {+ CU_CHECK(cuMemAddressReserve(&pool_addr, CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0));+ }-template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>-static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,- int ne0, int ne1, int ne2, int ne3,- int ne10, int ne11, int ne12, int ne13,- /*int s0, */ int s1, int s2, int s3,- /*int s10,*/ int s11, int s12, int s13) {+ // map at the end of the pool+ CU_CHECK(cuMemMap(pool_addr + pool_size, reserve_size, 0, handle, 0));- const int i = blockDim.x*blockIdx.x + threadIdx.x;+ // the memory allocation handle is no longer needed after mapping+ CU_CHECK(cuMemRelease(handle));- const int i3 = i/(ne2*ne1*ne0);- const int i2 = (i/(ne1*ne0)) % ne2;- const int i1 = (i/ne0) % ne1;- const int i0 = i % ne0;+ // set access+ CUmemAccessDesc access = {};+ access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;+ access.location.id = device;+ access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;+ CU_CHECK(cuMemSetAccess(pool_addr + pool_size, reserve_size, &access, 1));- if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {- return;- }+ // add to the pool+ pool_size += reserve_size;- const int i11 = i1 % ne11;- const int i12 = i2 % ne12;- const int i13 = i3 % ne13;+ //printf("cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\n",+ // id, (unsigned long long) (pool_size[id]/1024/1024),+ // (unsigned long long) (reserve_size/1024/1024));+ }- const size_t i_src0 = i3*s3 + i2*s2 + i1*s1;- const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;- const size_t i_dst = i_src0;+ GGML_ASSERT(pool_addr != 0);- const src0_t * src0_row = src0 + i_src0;- const src1_t * src1_row = src1 + i_src1;- dst_t * dst_row = dst + i_dst;+ void * ptr = (void *) (pool_addr + pool_used);+ *actual_size = size;+ pool_used += size;- const int i10 = i0 % ne10;- dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);-}+#ifdef DEBUG_CUDA_MALLOC+ printf("cuda pool[%d]: allocated %llu bytes at %llx\n", device, (unsigned long long) size, ptr);+#endif-static __global__ void acc_f32(const float * x, const float * y, float * dst, const int ne,- const int ne10, const int ne11, const int ne12,- const int nb1, const int nb2, int offset) {- const int i = blockDim.x * blockIdx.x + threadIdx.x;- if (i >= ne) {- return;- }- int src1_idx = i - offset;- int oz = src1_idx / nb2;- int oy = (src1_idx - (oz * nb2)) / nb1;- int ox = src1_idx % nb1;- if (src1_idx >= 0 && ox < ne10 && oy < ne11 && oz < ne12) {- dst[i] = x[i] + y[ox + oy * ne10 + oz * ne10 * ne11];- } else {- dst[i] = x[i];+ return ptr;}-}-static __global__ void gelu_f32(const float * x, float * dst, const int k) {- const float GELU_COEF_A = 0.044715f;- const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;- const int i = blockDim.x*blockIdx.x + threadIdx.x;+ void free(void * ptr, size_t size) override {+#ifdef DEBUG_CUDA_MALLOC+ printf("cuda pool[%d]: freed %llu bytes at %llx\n", device, (unsigned long long) size, ptr);+#endif- if (i >= k) {- return;+ pool_used -= size;++ // all deallocations must be in reverse order of the allocations+ GGML_ASSERT(ptr == (void *) (pool_addr + pool_used));}+};+#endif // !defined(GGML_USE_HIPBLAS)- float xi = x[i];- dst[i] = 0.5f*xi*(1.0f + tanhf(SQRT_2_OVER_PI*xi*(1.0f + GELU_COEF_A*xi*xi)));-}+template<typename T>+struct ggml_cuda_pool_alloc {+ ggml_cuda_pool * pool = nullptr;+ T * ptr = nullptr;+ size_t actual_size = 0;-static __global__ void silu_f32(const float * x, float * dst, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+ ggml_cuda_pool_alloc() = default;- if (i >= k) {- return;+ explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) {}- dst[i] = x[i] / (1.0f + expf(-x[i]));-}-static __global__ void gelu_quick_f32(const float * x, float * dst, int k) {- const float GELU_QUICK_COEF = -1.702f;- const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= k) {- return;+ ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) {+ alloc(size);}- dst[i] = x[i] * (1.0f / (1.0f + expf(GELU_QUICK_COEF * x[i])));-}-static __global__ void tanh_f32(const float * x, float * dst, int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= k) {- return;+ ~ggml_cuda_pool_alloc() {+ if (ptr != nullptr) {+ pool->free(ptr, actual_size);+ }}- dst[i] = tanhf(x[i]);-}--static __global__ void relu_f32(const float * x, float * dst, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= k) {- return;+ // size is in number of elements+ T * alloc(size_t size) {+ GGML_ASSERT(pool != nullptr);+ GGML_ASSERT(ptr == nullptr);+ ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);+ return ptr;}- dst[i] = fmaxf(x[i], 0);-}--static __global__ void hardsigmoid_f32(const float * x, float * dst, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= k) {- return;+ T * alloc(ggml_cuda_pool & pool, size_t size) {+ this->pool = &pool;+ return alloc(size);}- dst[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));-}--static __global__ void hardswish_f32(const float * x, float * dst, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= k) {- return;+ T * get() {+ return ptr;}- dst[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));-}-static __global__ void leaky_relu_f32(const float * x, float * dst, const int k, const float negative_slope) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= k) {- return;- }- dst[i] = fmaxf(x[i], 0) + fminf(x[i], 0.0f) * negative_slope;-}+ ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete;+ ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete;+ ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete;+ ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete;+};-static __global__ void sqr_f32(const float * x, float * dst, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= k) {- return;- }- dst[i] = x[i] * x[i];-}+// backend interface-template <int block_size>-static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {- const int row = blockIdx.x*blockDim.y + threadIdx.y;- const int tid = threadIdx.x;+struct ggml_backend_cuda_context {+ int device;+ std::string name;+ cudaEvent_t copy_event = nullptr;- float2 mean_var = make_float2(0.f, 0.f);+ cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } };+ cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};- for (int col = tid; col < ncols; col += block_size) {- const float xi = x[row*ncols + col];- mean_var.x += xi;- mean_var.y += xi * xi;+ explicit ggml_backend_cuda_context(int device) :+ device(device),+ name(GGML_CUDA_NAME + std::to_string(device)) {}- // sum up partial sums- mean_var = warp_reduce_sum(mean_var);- if (block_size > WARP_SIZE) {- __shared__ float2 s_sum[32];- int warp_id = threadIdx.x / WARP_SIZE;- int lane_id = threadIdx.x % WARP_SIZE;- if (lane_id == 0) {- s_sum[warp_id] = mean_var;+ ~ggml_backend_cuda_context() {+ if (copy_event != nullptr) {+ CUDA_CHECK(cudaEventDestroy(copy_event));+ }+ for (int i = 0; i < GGML_CUDA_MAX_DEVICES; ++i) {+ for (int j = 0; j < GGML_CUDA_MAX_STREAMS; ++j) {+ if (streams[i][j] != nullptr) {+ CUDA_CHECK(cudaStreamDestroy(streams[i][j]));+ }+ }+ if (cublas_handles[i] != nullptr) {+ CUBLAS_CHECK(cublasDestroy(cublas_handles[i]));+ }}- __syncthreads();- mean_var = s_sum[lane_id];- mean_var = warp_reduce_sum(mean_var);- }-- const float mean = mean_var.x / ncols;- const float var = mean_var.y / ncols - mean * mean;- const float inv_std = rsqrtf(var + eps);-- for (int col = tid; col < ncols; col += block_size) {- dst[row*ncols + col] = (x[row*ncols + col] - mean) * inv_std;}-}-static __global__ void concat_f32(const float * x,const float * y, float * dst, const int ne0, const int ne02) {- int nidx = threadIdx.x + blockIdx.x * blockDim.x;- if (nidx >= ne0) {- return;- }- // operation- int offset_dst =- nidx +- blockIdx.y * ne0 +- blockIdx.z * ne0 * gridDim.y;- if (blockIdx.z < ne02) { // src0- int offset_src =- nidx +- blockIdx.y * ne0 +- blockIdx.z * ne0 * gridDim.y;- dst[offset_dst] = x[offset_src];- } else {- int offset_src =- nidx +- blockIdx.y * ne0 +- (blockIdx.z - ne02) * ne0 * gridDim.y;- dst[offset_dst] = y[offset_src];+ cudaStream_t stream(int device, int stream) {+ if (streams[device][stream] == nullptr) {+ ggml_cuda_set_device(device);+ CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking));+ }+ return streams[device][stream];}-}-static __global__ void upscale_f32(const float * x, float * dst, const int ne00, const int nb02, const int scale_factor) {- int ne0 = ne00 * scale_factor;- int nidx = threadIdx.x + blockIdx.x * blockDim.x;- if (nidx >= ne0) {- return;+ cudaStream_t stream() {+ return stream(device, 0);}- // operation- int i00 = nidx / scale_factor;- int i01 = blockIdx.y / scale_factor;- int offset_src =- i00 +- i01 * ne00 +- blockIdx.z * nb02;- int offset_dst =- nidx +- blockIdx.y * ne0 +- blockIdx.z * ne0 * gridDim.y;- dst[offset_dst] = x[offset_src];-}-static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02) {- int nidx = threadIdx.x + blockIdx.x * blockDim.x;- if (nidx >= ne0) {- return;+ cublasHandle_t cublas_handle(int device) {+ if (cublas_handles[device] == nullptr) {+ ggml_cuda_set_device(device);+ CUBLAS_CHECK(cublasCreate(&cublas_handles[device]));+ CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH));+ }+ return cublas_handles[device];}- // operation- int offset_dst =- nidx +- blockIdx.y * ne0 +- blockIdx.z * ne0 * gridDim.y;- if (nidx < ne00 && blockIdx.y < ne01 && blockIdx.z < ne02) {- int offset_src =- nidx +- blockIdx.y * ne00 +- blockIdx.z * ne00 * ne01;- dst[offset_dst] = x[offset_src];- } else {- dst[offset_dst] = 0.0f;+ cublasHandle_t cublas_handle() {+ return cublas_handle(device);}-}--template <int block_size>-static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {- int start = blockIdx.x * group_size;- int end = start + group_size;- start += threadIdx.x;+ // pool+ std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES];- if (end >= ne_elements) {- end = ne_elements;+ static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device) {+#if !defined(GGML_USE_HIPBLAS)+ if (get_cuda_global_info().devices[device].vmm) {+ return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_vmm(device));+ }+#endif+ return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(device));}- float tmp = 0.0f; // partial sum for thread in warp-- for (int j = start; j < end; j += block_size) {- tmp += x[j];+ ggml_cuda_pool & pool(int device) {+ if (pools[device] == nullptr) {+ pools[device] = new_pool_for_device(device);+ }+ return *pools[device];}- tmp = warp_reduce_sum(tmp);- if (block_size > WARP_SIZE) {- __shared__ float s_sum[32];- int warp_id = threadIdx.x / WARP_SIZE;- int lane_id = threadIdx.x % WARP_SIZE;- if (lane_id == 0) {- s_sum[warp_id] = tmp;- }- __syncthreads();- tmp = s_sum[lane_id];- tmp = warp_reduce_sum(tmp);+ ggml_cuda_pool & pool() {+ return pool(device);}+};- float mean = tmp / group_size;- tmp = 0.0f;+// cuda buffer- for (int j = start; j < end; j += block_size) {- float xi = x[j] - mean;- dst[j] = xi;- tmp += xi * xi;- }+struct ggml_backend_cuda_buffer_context {+ int device;+ void * dev_ptr = nullptr;+ std::string name;- tmp = warp_reduce_sum(tmp);- if (block_size > WARP_SIZE) {- __shared__ float s_sum[32];- int warp_id = threadIdx.x / WARP_SIZE;- int lane_id = threadIdx.x % WARP_SIZE;- if (lane_id == 0) {- s_sum[warp_id] = tmp;- }- __syncthreads();- tmp = s_sum[lane_id];- tmp = warp_reduce_sum(tmp);+ ggml_backend_cuda_buffer_context(int device, void * dev_ptr) :+ device(device), dev_ptr(dev_ptr),+ name(GGML_CUDA_NAME + std::to_string(device)) {}- float variance = tmp / group_size;- float scale = rsqrtf(variance + eps);- for (int j = start; j < end; j += block_size) {- dst[j] *= scale;+ ~ggml_backend_cuda_buffer_context() {+ CUDA_CHECK(cudaFree(dev_ptr));}+};++GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ return ctx->name.c_str();}-template <int block_size>-static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {- const int row = blockIdx.x*blockDim.y + threadIdx.y;- const int tid = threadIdx.x;+GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {+ return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name;+}- float tmp = 0.0f; // partial sum for thread in warp+GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ delete ctx;+}- for (int col = tid; col < ncols; col += block_size) {- const float xi = x[row*ncols + col];- tmp += xi * xi;- }+GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ return ctx->dev_ptr;+}- // sum up partial sums- tmp = warp_reduce_sum(tmp);- if (block_size > WARP_SIZE) {- __shared__ float s_sum[32];- int warp_id = threadIdx.x / WARP_SIZE;- int lane_id = threadIdx.x % WARP_SIZE;- if (lane_id == 0) {- s_sum[warp_id] = tmp;- }- __syncthreads();- tmp = s_sum[lane_id];- tmp = warp_reduce_sum(tmp);+GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;++ if (tensor->view_src != NULL && tensor->view_offs == 0) {+ assert(tensor->view_src->buffer->buft == buffer->buft);+ tensor->backend = tensor->view_src->backend;+ tensor->extra = tensor->view_src->extra;+ return;}- const float mean = tmp / ncols;- const float scale = rsqrtf(mean + eps);+ if (ggml_is_quantized(tensor->type)) {+ // initialize padding to 0 to avoid possible NaN values+ size_t original_size = ggml_nbytes(tensor);+ size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);- for (int col = tid; col < ncols; col += block_size) {- dst[row*ncols + col] = scale * x[row*ncols + col];+ if (padded_size > original_size && tensor->view_src == nullptr) {+ ggml_cuda_set_device(ctx->device);+ CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));+ }}}-static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int ib, const int iqs, dfloat2 & v){- const block_q4_0 * x = (const block_q4_0 *) vx;-- const dfloat d = x[ib].d;+GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;- const int vui = x[ib].qs[iqs];+ ggml_cuda_set_device(ctx->device);+ CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cudaStreamPerThread));+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));+}- v.x = vui & 0xF;- v.y = vui >> 4;+GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;-#ifdef GGML_CUDA_F16- v = __hsub2(v, {8.0f, 8.0f});- v = __hmul2(v, {d, d});-#else- v.x = (v.x - 8.0f) * d;- v.y = (v.y - 8.0f) * d;-#endif // GGML_CUDA_F16+ ggml_cuda_set_device(ctx->device);+ CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cudaStreamPerThread));+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}-static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int ib, const int iqs, dfloat2 & v){- const block_q4_1 * x = (const block_q4_1 *) vx;-- const dfloat d = __low2half(x[ib].dm);- const dfloat m = __high2half(x[ib].dm);+GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {+ if (ggml_backend_buffer_is_cuda(src->buffer)) {+ ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;+ ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;+ if (src_ctx->device == dst_ctx->device) {+ CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread));+ } else {+ CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread));+ }+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));+ return true;+ }+ return false;- const int vui = x[ib].qs[iqs];+ GGML_UNUSED(buffer);+}- v.x = vui & 0xF;- v.y = vui >> 4;+GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;-#ifdef GGML_CUDA_F16- v = __hmul2(v, {d, d});- v = __hadd2(v, {m, m});-#else- v.x = (v.x * d) + m;- v.y = (v.y * d) + m;-#endif // GGML_CUDA_F16+ ggml_cuda_set_device(ctx->device);+ CUDA_CHECK(cudaDeviceSynchronize());+ CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size));+ CUDA_CHECK(cudaDeviceSynchronize());}-static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int ib, const int iqs, dfloat2 & v){- const block_q5_0 * x = (const block_q5_0 *) vx;+static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {+ /* .get_name = */ ggml_backend_cuda_buffer_get_name,+ /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,+ /* .get_base = */ ggml_backend_cuda_buffer_get_base,+ /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor,+ /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,+ /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,+ /* .clear = */ ggml_backend_cuda_buffer_clear,+ /* .reset = */ NULL,+};- const dfloat d = x[ib].d;-- uint32_t qh;- memcpy(&qh, x[ib].qh, sizeof(qh));-- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;+// cuda buffer type+struct ggml_backend_cuda_buffer_type_context {+ int device;+ std::string name;+};- v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);- v.y = ((x[ib].qs[iqs] >> 4) | xh_1);+GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) {+ ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;-#ifdef GGML_CUDA_F16- v = __hsub2(v, {16.0f, 16.0f});- v = __hmul2(v, {d, d});-#else- v.x = (v.x - 16.0f) * d;- v.y = (v.y - 16.0f) * d;-#endif // GGML_CUDA_F16+ return ctx->name.c_str();}-static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int ib, const int iqs, dfloat2 & v){- const block_q5_1 * x = (const block_q5_1 *) vx;+GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;- const dfloat d = __low2half(x[ib].dm);- const dfloat m = __high2half(x[ib].dm);+ ggml_cuda_set_device(buft_ctx->device);- uint32_t qh;- memcpy(&qh, x[ib].qh, sizeof(qh));+ size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;+ void * dev_ptr;+ cudaError_t err = cudaMalloc(&dev_ptr, size);+ if (err != cudaSuccess) {+ fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err));+ return nullptr;+ }- v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);- v.y = ((x[ib].qs[iqs] >> 4) | xh_1);+ ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr);-#ifdef GGML_CUDA_F16- v = __hmul2(v, {d, d});- v = __hadd2(v, {m, m});-#else- v.x = (v.x * d) + m;- v.y = (v.y * d) + m;-#endif // GGML_CUDA_F16+ return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);}-static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int ib, const int iqs, dfloat2 & v){- const block_q8_0 * x = (const block_q8_0 *) vx;+GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+ return 128;- const dfloat d = x[ib].d;+ GGML_UNUSED(buft);+}- v.x = x[ib].qs[iqs + 0];- v.y = x[ib].qs[iqs + 1];+GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {+ size_t size = ggml_nbytes(tensor);+ int64_t ne0 = tensor->ne[0];-#ifdef GGML_CUDA_F16- v = __hmul2(v, {d, d});-#else- v.x *= d;- v.y *= d;-#endif // GGML_CUDA_F16-}+ if (ggml_is_quantized(tensor->type)) {+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }+ }-template<typename dst_t>-static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {+ return size;- const int i = blockIdx.x;+ GGML_UNUSED(buft);+}- // assume 32 threads- const int tid = threadIdx.x;- const int il = tid/8;- const int ir = tid%8;- const int ib = 8*i + ir;- if (ib >= nb32) {- return;+GGML_CALL static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ if (!ggml_backend_is_cuda(backend)) {+ return false;}- dst_t * y = yy + 256*i + 32*ir + 4*il;+ ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;- const block_q4_0 * x = (const block_q4_0 *)vx + ib;- const float d = __half2float(x->d);- const float dm = -8*d;+ return buft_ctx->device == cuda_ctx->device;+}- const uint8_t * q = x->qs + 4*il;+static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {+ /* .get_name = */ ggml_backend_cuda_buffer_type_name,+ /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX+ /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,+ /* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend,+ /* .is_host = */ NULL,+};- for (int l = 0; l < 4; ++l) {- y[l+ 0] = d * (q[l] & 0xF) + dm;- y[l+16] = d * (q[l] >> 4) + dm;+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {+ static std::mutex mutex;+ std::lock_guard<std::mutex> lock(mutex);++ if (device >= ggml_backend_cuda_get_device_count()) {+ return nullptr;}-}-template<typename dst_t>-static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {+ static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];- const int i = blockIdx.x;+ static bool ggml_backend_cuda_buffer_type_initialized = false;- // assume 32 threads- const int tid = threadIdx.x;- const int il = tid/8;- const int ir = tid%8;- const int ib = 8*i + ir;- if (ib >= nb32) {- return;+ if (!ggml_backend_cuda_buffer_type_initialized) {+ for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) {+ ggml_backend_cuda_buffer_types[i] = {+ /* .iface = */ ggml_backend_cuda_buffer_type_interface,+ /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)},+ };+ }+ ggml_backend_cuda_buffer_type_initialized = true;}- dst_t * y = yy + 256*i + 32*ir + 4*il;+ return &ggml_backend_cuda_buffer_types[device];+}- const block_q4_1 * x = (const block_q4_1 *)vx + ib;- const float2 d = __half22float2(x->dm);+// cuda split buffer- const uint8_t * q = x->qs + 4*il;+static int64_t get_row_rounding(ggml_type type, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split) {+ int64_t min_compute_capability = INT_MAX;+ int64_t max_compute_capability = INT_MIN;+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ if (tensor_split[id] < (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {+ if (min_compute_capability > get_cuda_global_info().devices[id].cc) {+ min_compute_capability = get_cuda_global_info().devices[id].cc;+ }+ if (max_compute_capability < get_cuda_global_info().devices[id].cc) {+ max_compute_capability = get_cuda_global_info().devices[id].cc;+ }+ }+ }- for (int l = 0; l < 4; ++l) {- y[l+ 0] = d.x * (q[l] & 0xF) + d.y;- y[l+16] = d.x * (q[l] >> 4) + d.y;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ switch(type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ return max_compute_capability >= CC_RDNA2 ? 128 : 64;+ case GGML_TYPE_F16:+ case GGML_TYPE_F32:+ return 1;+ case GGML_TYPE_Q2_K:+ return max_compute_capability >= CC_RDNA2 ? 128 : 32;+ case GGML_TYPE_Q3_K:+ return min_compute_capability < CC_RDNA2 ? 128 : 64;+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_IQ4_XS:+ case GGML_TYPE_IQ3_S:+ return max_compute_capability >= CC_RDNA2 ? 128 : 64;+ default:+ GGML_ASSERT(false);+ }+#else+ switch(type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ return max_compute_capability >= CC_VOLTA ? 128 : 64;+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ return 64;+ case GGML_TYPE_F16:+ case GGML_TYPE_F32:+ return 1;+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_IQ4_XS:+ case GGML_TYPE_IQ3_S:+ return max_compute_capability >= CC_VOLTA ? 128 : 64;+ case GGML_TYPE_Q6_K:+ return 64;+ default:+ GGML_ASSERT(false);}+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)}-//================================== k-quants+static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split, int id) {+ const int64_t nrows = ggml_nrows(tensor);+ const int64_t rounding = get_row_rounding(tensor->type, tensor_split);-template<typename dst_t>-static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ *row_low = id == 0 ? 0 : nrows*tensor_split[id];+ *row_low -= *row_low % rounding;- const int i = blockIdx.x;- const block_q2_K * x = (const block_q2_K *) vx;+ if (id == ggml_backend_cuda_get_device_count() - 1) {+ *row_high = nrows;+ } else {+ *row_high = nrows*tensor_split[id + 1];+ *row_high -= *row_high % rounding;+ }+}- const int tid = threadIdx.x;-#if QK_K == 256- const int n = tid/32;- const int l = tid - 32*n;- const int is = 8*n + l/16;+static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");- const uint8_t q = x[i].qs[32*n + l];- dst_t * y = yy + i*QK_K + 128*n;+ return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]);+}- float dall = __low2half(x[i].dm);- float dmin = __high2half(x[i].dm);- y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);- y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);- y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);- y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);-#else- const int is = tid/16; // 0 or 1- const int il = tid%16; // 0...15- const uint8_t q = x[i].qs[il] >> (2*is);- dst_t * y = yy + i*QK_K + 16*is + il;- float dall = __low2half(x[i].dm);- float dmin = __high2half(x[i].dm);- y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);- y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4);-#endif+struct ggml_backend_cuda_split_buffer_type_context {+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;+};-}+struct ggml_backend_cuda_split_buffer_context {+ ~ggml_backend_cuda_split_buffer_context() {+ for (ggml_tensor_extra_gpu * extra : tensor_extras) {+ for (int id = 0; id < GGML_CUDA_MAX_DEVICES; ++id) {+ for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {+ if (extra->events[id][is] != nullptr) {+ CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));+ }+ }+ if (extra->data_device[id] != nullptr) {+ CUDA_CHECK(cudaFree(extra->data_device[id]));+ }+ }+ delete extra;+ }+ }-template<typename dst_t>-static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ std::vector<ggml_tensor_extra_gpu *> tensor_extras;+};- const int i = blockIdx.x;- const block_q3_K * x = (const block_q3_K *) vx;+GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) {+ return GGML_CUDA_NAME "_Split";-#if QK_K == 256- const int r = threadIdx.x/4;- const int tid = r/2;- const int is0 = r%2;- const int l0 = 16*is0 + 4*(threadIdx.x%4);- const int n = tid / 4;- const int j = tid - 4*n;+ GGML_UNUSED(buffer);+}- uint8_t m = 1 << (4*n + j);- int is = 8*n + 2*j + is0;- int shift = 2*j;+static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {+ return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name;+ GGML_UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds+}- int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :- is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :- is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :- (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);- float d_all = x[i].d;- float dl = d_all * (us - 32);+GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;+ delete ctx;+}- dst_t * y = yy + i*QK_K + 128*n + 32*j;- const uint8_t * q = x[i].qs + 32*n;- const uint8_t * hm = x[i].hmask;+GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {+ // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced+ return (void *)0x1000;- for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));-#else- const int tid = threadIdx.x;- const int is = tid/16; // 0 or 1- const int il = tid%16; // 0...15- const int im = il/8; // 0...1- const int in = il%8; // 0...7+ GGML_UNUSED(buffer);+}- dst_t * y = yy + i*QK_K + 16*is + il;+GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {+ GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported- const uint8_t q = x[i].qs[il] >> (2*is);- const uint8_t h = x[i].hmask[in] >> (2*is + im);- const float d = (float)x[i].d;+ ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;- if (is == 0) {- y[ 0] = d * ((x[i].scales[0] & 0xF) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));- y[32] = d * ((x[i].scales[1] & 0xF) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));- } else {- y[ 0] = d * ((x[i].scales[0] >> 4) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));- y[32] = d * ((x[i].scales[1] >> 4) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));- }-#endif+ const int64_t ne0 = tensor->ne[0];-}+ ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};+ ctx->tensor_extras.push_back(extra);-#if QK_K == 256-static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {- if (j < 4) {- d = q[j] & 63; m = q[j + 4] & 63;- } else {- d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);- m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);- }-}-#endif+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);-template<typename dst_t>-static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const block_q4_K * x = (const block_q4_K *) vx;+ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }- const int i = blockIdx.x;+ size_t size = ggml_nbytes_split(tensor, nrows_split);+ const size_t original_size = size;-#if QK_K == 256- // assume 32 threads- const int tid = threadIdx.x;- const int il = tid/8;- const int ir = tid%8;- const int is = 2*il;- const int n = 4;+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }- dst_t * y = yy + i*QK_K + 64*il + n*ir;+ // FIXME: do not crash if cudaMalloc fails+ // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first+ ggml_cuda_set_device(id);+ char * buf;+ CUDA_CHECK(cudaMalloc(&buf, size));- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ // set padding to 0 to avoid possible NaN values+ if (size > original_size) {+ CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size));+ }- const uint8_t * q = x[i].qs + 32*il + n*ir;+ extra->data_device[id] = buf;- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, sc, m);- const float d1 = dall * sc; const float m1 = dmin * m;- get_scale_min_k4(is + 1, x[i].scales, sc, m);- const float d2 = dall * sc; const float m2 = dmin * m;- for (int l = 0; l < n; ++l) {- y[l + 0] = d1 * (q[l] & 0xF) - m1;- y[l +32] = d2 * (q[l] >> 4) - m2;+ for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {+ CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming));+ }}-#else- const int tid = threadIdx.x;- const uint8_t * q = x[i].qs;- dst_t * y = yy + i*QK_K;- const float d = (float)x[i].dm[0];- const float m = (float)x[i].dm[1];- y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4);- y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4);-#endif+ tensor->extra = extra;}-template<typename dst_t>-static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const block_q5_K * x = (const block_q5_K *) vx;+GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ // split tensors must always be set in their entirety at once+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(size == ggml_nbytes(tensor));- const int i = blockIdx.x;+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;-#if QK_K == 256- // assume 64 threads - this is very slightly better than the one below- const int tid = threadIdx.x;- const int il = tid/16; // il is in 0...3- const int ir = tid%16; // ir is in 0...15- const int is = 2*il; // is is in 0...6+ const int64_t ne0 = tensor->ne[0];+ const size_t nb1 = tensor->nb[1];+ ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;- dst_t * y = yy + i*QK_K + 64*il + 2*ir;+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }- const uint8_t * ql = x[i].qs + 32*il + 2*ir;- const uint8_t * qh = x[i].qh + 2*ir;+ const size_t offset_split = row_low*nb1;+ size_t size = ggml_nbytes_split(tensor, nrows_split);+ const size_t original_size = size;- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, sc, m);- const float d1 = dall * sc; const float m1 = dmin * m;- get_scale_min_k4(is + 1, x[i].scales, sc, m);- const float d2 = dall * sc; const float m2 = dmin * m;+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }- uint8_t hm = 1 << (2*il);- y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;- y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;- hm <<= 1;- y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;- y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;-#else- const int tid = threadIdx.x;- const uint8_t q = x[i].qs[tid];- const int im = tid/8; // 0...3- const int in = tid%8; // 0...7- const int is = tid/16; // 0 or 1- const uint8_t h = x[i].qh[in] >> im;- const float d = x[i].d;- dst_t * y = yy + i*QK_K + tid;- y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16));- y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16));-#endif+ const char * buf_host = (const char *)data + offset_split;+ CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread));+ }++ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));+ }}-template<typename dst_t>-static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const block_q6_K * x = (const block_q6_K *) vx;+GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ // split tensors must always be set in their entirety at once+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(size == ggml_nbytes(tensor));- const int i = blockIdx.x;-#if QK_K == 256+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;- // assume 64 threads - this is very slightly better than the one below- const int tid = threadIdx.x;- const int ip = tid/32; // ip is 0 or 1- const int il = tid - 32*ip; // 0...32- const int is = 8*ip + il/16;+ const int64_t ne0 = tensor->ne[0];+ const size_t nb1 = tensor->nb[1];+ ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;- dst_t * y = yy + i*QK_K + 128*ip + il;+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);- const float d = x[i].d;+ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }- const uint8_t * ql = x[i].ql + 64*ip + il;- const uint8_t qh = x[i].qh[32*ip + il];- const int8_t * sc = x[i].scales + is;+ const size_t offset_split = row_low*nb1;+ size_t size = ggml_nbytes_split(tensor, nrows_split);+ const size_t original_size = size;- y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);- y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);- y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);- y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);-#else+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }- // assume 32 threads- const int tid = threadIdx.x;- const int ip = tid/16; // 0 or 1- const int il = tid - 16*ip; // 0...15-- dst_t * y = yy + i*QK_K + 16*ip + il;-- const float d = x[i].d;-- const uint8_t ql = x[i].ql[16*ip + il];- const uint8_t qh = x[i].qh[il] >> (2*ip);- const int8_t * sc = x[i].scales;+ char * buf_host = (char *)data + offset_split;+ CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread));+ }- y[ 0] = d * sc[ip+0] * ((int8_t)((ql & 0xF) | (((qh >> 0) & 3) << 4)) - 32);- y[32] = d * sc[ip+2] * ((int8_t)((ql >> 4) | (((qh >> 4) & 3) << 4)) - 32);-#endif+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));+ }}-static const __device__ uint64_t iq2xxs_grid[256] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,- 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,- 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,- 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,- 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,- 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,- 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,- 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,- 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,- 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,- 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,- 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,- 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,- 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,- 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,- 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,- 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,- 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,- 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,- 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,- 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,- 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,- 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,- 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,- 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,- 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,- 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,- 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,- 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,- 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,- 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,- 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,- 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,- 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,- 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,- 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,- 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,- 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,- 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,- 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,- 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,- 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,- 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,- 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,- 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,- 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,- 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,- 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,- 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,- 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,- 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,- 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,- 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,- 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,- 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,- 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,- 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,- 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,- 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,- 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,- 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,- 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,- 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,-};--static const __device__ uint64_t iq2xs_grid[512] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,- 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,- 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,- 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,- 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,- 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,- 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,- 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,- 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,- 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,- 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,- 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,- 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,- 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,- 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,- 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,- 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,- 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,- 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,- 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,- 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,- 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,- 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,- 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,- 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,- 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,- 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,- 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,- 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,- 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,- 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,- 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,- 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,- 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,- 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,- 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,- 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,- 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,- 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,- 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,- 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,- 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,- 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,- 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,- 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,- 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,- 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,- 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,- 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,- 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,- 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,- 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,- 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,- 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,- 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,- 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,- 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,- 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,- 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,- 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,- 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,- 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,- 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,- 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,- 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,- 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,- 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,- 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,- 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,- 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,- 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,- 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,- 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,- 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,- 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,- 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,- 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,- 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,- 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,- 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,- 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,- 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,- 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,- 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,- 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,- 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,- 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,- 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,- 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,- 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,- 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,- 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,- 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,- 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,- 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,- 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,- 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,- 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,- 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,- 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,- 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,- 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,- 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,- 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,- 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,- 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,- 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,- 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,- 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,- 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,- 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,- 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,- 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,- 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,- 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,- 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,- 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,- 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,- 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,- 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,- 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,- 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,-};--static const __device__ uint64_t iq2s_grid[1024] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,- 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,- 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,- 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,- 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,- 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,- 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,- 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,- 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,- 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,- 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,- 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,- 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,- 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,- 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,- 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,- 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,- 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,- 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,- 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,- 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,- 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,- 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,- 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,- 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,- 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,- 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,- 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,- 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,- 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,- 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,- 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,- 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,- 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,- 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,- 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,- 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,- 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,- 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,- 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,- 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,- 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,- 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,- 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,- 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,- 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,- 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,- 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,- 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,- 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,- 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,- 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,- 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,- 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,- 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,- 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,- 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,- 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,- 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,- 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,- 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,- 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,- 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,- 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,- 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,- 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,- 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,- 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,- 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,- 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,- 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,- 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,- 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,- 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,- 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,- 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,- 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,- 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,- 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,- 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,- 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,- 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,- 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,- 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,- 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,- 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,- 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,- 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,- 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,- 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,- 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,- 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,- 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,- 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,- 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,- 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,- 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,- 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,- 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,- 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,- 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,- 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,- 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,- 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,- 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,- 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,- 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,- 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,- 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,- 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,- 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,- 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,- 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,- 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,- 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,- 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,- 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,- 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,- 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,- 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,- 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,- 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,- 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,- 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,- 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,- 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,- 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,- 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,- 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,- 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,- 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,- 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,- 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,- 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,- 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,- 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,- 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,- 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,- 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,- 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,- 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,- 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,- 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,- 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,- 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,- 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,- 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,- 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,- 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,- 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,- 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,- 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,- 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,- 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,- 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,- 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,- 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,- 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,- 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,- 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,- 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,- 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,- 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,- 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,- 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,- 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,- 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,- 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,- 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,- 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,- 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,- 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,- 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,- 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,- 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,- 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,- 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,- 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,- 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,- 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,- 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,- 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,- 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,- 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,- 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,- 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,- 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,- 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,- 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,- 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,- 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,- 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,- 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,- 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,- 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,- 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,- 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,- 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,- 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,- 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,- 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,- 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,- 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,- 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,- 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,- 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,- 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,- 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,- 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,- 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,- 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,- 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,- 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,- 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,- 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,- 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,- 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,- 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,- 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,- 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,- 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,- 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,- 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,- 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,- 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,- 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,- 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,- 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,- 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,- 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,- 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,- 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,- 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,- 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,- 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,- 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,- 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,- 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,- 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,- 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,- 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,- 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,- 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,- 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,- 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,- 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,- 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,- 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,- 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,- 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,- 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,- 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,- 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,-};+GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ GGML_UNUSED(buffer);+ GGML_UNUSED(value);+}-static const __device__ uint32_t iq3xxs_grid[256] = {- 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,- 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,- 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,- 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,- 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,- 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,- 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,- 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,- 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,- 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,- 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,- 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,- 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,- 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,- 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,- 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,- 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,- 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,- 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,- 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,- 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,- 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,- 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,- 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,- 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,- 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,- 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,- 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,- 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,- 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,- 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,- 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,+static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {+ /* .get_name = */ ggml_backend_cuda_split_buffer_get_name,+ /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer,+ /* .get_base = */ ggml_backend_cuda_split_buffer_get_base,+ /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor,+ /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor,+ /* .cpy_tensor = */ NULL,+ /* .clear = */ ggml_backend_cuda_split_buffer_clear,+ /* .reset = */ NULL,};-static const __device__ uint32_t iq3xs_grid[512] = {- 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14,- 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414,- 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24,- 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c,- 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c,- 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34,- 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c,- 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414,- 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,- 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404,- 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434,- 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c,- 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404,- 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414,- 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414,- 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404,- 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c,- 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,- 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404,- 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e,- 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14,- 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c,- 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424,- 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c,- 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c,- 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e,- 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,- 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e,- 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424,- 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e,- 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424,- 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404,- 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c,- 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e,- 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c,- 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,- 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c,- 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404,- 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04,- 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c,- 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414,- 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c,- 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c,- 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424,- 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,- 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c,- 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414,- 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c,- 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e,- 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04,- 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424,- 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14,- 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34,- 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,- 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434,- 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c,- 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424,- 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24,- 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24,- 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e,- 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c,- 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c,- 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,- 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,-};+// cuda split buffer type+GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) {+ return GGML_CUDA_NAME "_Split";-static const __device__ uint64_t iq1s_grid[512] = {- 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,- 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,- 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,- 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,- 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,- 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,- 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,- 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,- 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,- 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,- 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,- 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,- 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,- 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,- 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,- 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,- 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,- 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,- 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,- 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,- 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,- 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,- 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,- 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,- 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,- 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,- 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,- 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,- 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,- 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,- 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,- 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,- 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,- 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,- 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,- 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,- 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,- 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,- 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,- 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,- 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,- 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,- 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,- 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,- 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,- 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,- 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,- 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,- 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,- 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,- 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,- 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,- 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,- 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,- 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,- 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,- 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,- 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,- 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,- 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,- 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,- 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,- 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,- 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,- 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,- 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,- 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,- 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,- 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,- 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,- 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,- 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,- 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,- 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,- 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,- 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,- 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,- 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,- 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,- 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,- 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,- 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,- 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,- 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,- 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,- 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,- 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,- 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,- 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,- 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,- 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,- 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,- 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,- 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,- 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,- 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,- 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,- 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,- 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,- 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,- 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,- 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,- 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,- 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,- 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,- 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,- 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,- 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,- 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,- 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,- 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,- 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,- 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,- 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,- 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,- 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,- 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,- 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,- 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,- 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,- 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,- 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,- 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,- 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,- 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,- 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,- 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,- 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,-};+ GGML_UNUSED(buft);+}-static const __device__ uint8_t ksigns_iq2xs[128] = {- 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,- 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,- 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,- 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,- 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,- 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,- 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,- 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,-};+GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point+ // instead, we allocate them for each tensor separately in init_tensor+ // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,+ // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct.+ ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context();-//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-static const __device__ uint64_t ksigns64[128] = {- 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,- 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,- 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,- 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,- 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,- 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,- 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,- 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,- 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,- 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,- 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,- 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,- 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,- 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,- 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,- 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,- 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,- 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,- 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,- 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,- 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,- 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,- 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,- 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,- 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,- 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,- 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,- 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,- 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,- 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,- 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,- 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,-};-//#endif+ return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);+}-static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};+GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+ return 128;-inline bool ggml_cuda_supports_mmq(enum ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- return true;- default:- return false;- }+ GGML_UNUSED(buft);}-template<typename dst_t>-static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {+ ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;- const int i = blockIdx.x;- const block_iq2_xxs * x = (const block_iq2_xxs *) vx;+ size_t total_size = 0;- const int tid = threadIdx.x;-#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint16_t * q2 = x[i].qs + 4*ib;- const uint8_t * aux8 = (const uint8_t *)q2;- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);- const uint32_t aux32 = q2[2] | (q2[3] << 16);- const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-#else- assert(false);-#endif+ const int64_t ne0 = tensor->ne[0];-}+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id);-template<typename dst_t>-static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }- const int i = blockIdx.x;- const block_iq2_xs * x = (const block_iq2_xs *) vx;+ total_size += ggml_nbytes_split(tensor, nrows_split);- const int tid = threadIdx.x;-#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint16_t * q2 = x[i].qs + 4*ib;- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));- const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;- const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-#else- assert(false);-#endif+ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }+ }+ return total_size;}-template<typename dst_t>-static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {+GGML_CALL static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ return ggml_backend_is_cuda(backend);- const int i = blockIdx.x;- const block_iq2_s * x = (const block_iq2_s *) vx;+ GGML_UNUSED(buft);+}- const int tid = threadIdx.x;-#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));- const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;- const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-#else- assert(false);-#endif+GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {+ return false;+ GGML_UNUSED(buft);}-template<typename dst_t>-static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {-- const int i = blockIdx.x;- const block_iq3_xxs * x = (const block_iq3_xxs *) vx;+static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = {+ /* .get_name = */ ggml_backend_cuda_split_buffer_type_name,+ /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX+ /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,+ /* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend,+ /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,+};- const int tid = threadIdx.x;-#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint8_t * q3 = x[i].qs + 8*ib;- const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;- const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);- const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);- const uint32_t aux32 = gas[0] | (gas[1] << 16);- const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];- for (int j = 0; j < 4; ++j) {- y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);- y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);- }-#else- assert(false);-#endif+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) {+ static std::mutex mutex;+ std::lock_guard<std::mutex> lock(mutex);-}+ static std::map<std::array<float, GGML_CUDA_MAX_DEVICES>, struct ggml_backend_buffer_type> buft_map;-template<typename dst_t>-static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split_arr = {};- const int i = blockIdx.x;- const block_iq3_s * x = (const block_iq3_s *) vx;+ bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; });+ if (all_zero) {+ tensor_split_arr = get_cuda_global_info().default_tensor_split;+ } else {+ float split_sum = 0.0f;+ for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {+ tensor_split_arr[i] = split_sum;+ split_sum += tensor_split[i];+ }+ for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {+ tensor_split_arr[i] /= split_sum;+ }+ }- const int tid = threadIdx.x;-#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint8_t * qs = x[i].qs + 8*ib;- const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));- const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));- const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)) * 0.5f;- const uint8_t signs = x[i].signs[4*ib + il];- for (int j = 0; j < 4; ++j) {- y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);- y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);+ auto it = buft_map.find(tensor_split_arr);+ if (it != buft_map.end()) {+ return &it->second;}-#else- assert(false);-#endif-}+ struct ggml_backend_buffer_type buft {+ /* .iface = */ ggml_backend_cuda_split_buffer_type_interface,+ /* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr},+ };-template<typename dst_t>-static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ auto result = buft_map.emplace(tensor_split_arr, buft);+ return &result.first->second;+}- const int i = blockIdx.x;- const block_iq1_s * x = (const block_iq1_s *) vx;+// host buffer type- const int tid = threadIdx.x;-#if QK_K == 256- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const int i8 = 4*ib+il;- uint8_t h = x[i].scales[i8/2] >> 4*(i8%2);- const int8_t * grid = (const int8_t *)(iq1s_grid + (x[i].qs[i8] | ((h & 8) << 5)));- const float d = (float)x[i].d * (2*(h & 7) + 1);- for (int j = 0; j < 8; ++j) y[j] = d * grid[j];-#else- assert(false);-#endif+GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) {+ return GGML_CUDA_NAME "_Host";+ GGML_UNUSED(buft);}-static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};+GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) {+ return GGML_CUDA_NAME "_Host";-template<typename dst_t>-static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ GGML_UNUSED(buffer);+}- const int i = blockIdx.x;- const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);+GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ CUDA_CHECK(cudaFreeHost(buffer->context));+}- const int tid = threadIdx.x;- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 4*il;- const uint8_t * q4 = x[ib].qs + 4*il;- const float d = (float)x[ib].d;- for (int j = 0; j < 4; ++j) {- y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];- y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];+static void * ggml_cuda_host_malloc(size_t size) {+ if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {+ return nullptr;}-}--template<typename dst_t>-static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {-- const int i = blockIdx.x;- const block_iq4_xs * x = (const block_iq4_xs *)vx;-- const int tid = threadIdx.x;- const int il = tid/8; // 0...3- const int ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 4*il;- const uint8_t * q4 = x[i].qs + 16*ib + 4*il;- const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);- for (int j = 0; j < 4; ++j) {- y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];- y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];+ void * ptr = nullptr;+ cudaError_t err = cudaMallocHost((void **) &ptr, size);+ if (err != cudaSuccess) {+ // clear the error+ cudaGetLastError();+ fprintf(stderr, "%s: warning: failed to allocate %.2f MiB of pinned memory: %s\n", __func__,+ size/1024.0/1024.0, cudaGetErrorString(err));+ return nullptr;}+ return ptr;}-static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {+GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ void * ptr = ggml_cuda_host_malloc(size);- static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");+ if (ptr == nullptr) {+ // fallback to cpu buffer+ return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);+ }- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;+ ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);+ buffer->buft = buft;+ buffer->iface.get_name = ggml_backend_cuda_host_buffer_name;+ buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+ return buffer;+}- const block_q2_K * x = (const block_q2_K *)vx + ib0;+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {+ static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {+ /* .iface = */ {+ /* .get_name = */ ggml_backend_cuda_host_buffer_type_name,+ /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX+ /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,+ /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend,+ /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,+ },+ /* .context = */ nullptr,+ };- float tmp = 0; // partial sum for thread in warp+ return &ggml_backend_cuda_buffer_type_host;+}-#if QK_K == 256- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...15- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1+//static bool ggml_backend_buffer_is_cuda_host(ggml_backend_buffer_t buffer) {+// return buffer->buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;+//}- const int step = 16/K_QUANTS_PER_ITERATION;- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0...15 or 0...7- const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2- const int q_offset = 32*im + l0;- const int s_offset = 8*im;- const int y_offset = 128*im + l0;+/// kernels- uint32_t aux[4];- const uint8_t * d = (const uint8_t *)aux;- const uint8_t * m = (const uint8_t *)(aux + 2);- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+#if defined(GGML_USE_HIPBLAS)+#define __CUDA_ARCH__ 1300- const float * y = yy + i * QK_K + y_offset;- const uint8_t * q = x[i].qs + q_offset;+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \+ defined(__gfx1150__) || defined(__gfx1151__)+#define RDNA3+#endif- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \+ defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)+#define RDNA2+#endif- const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset);- aux[0] = a[0] & 0x0f0f0f0f;- aux[1] = a[1] & 0x0f0f0f0f;- aux[2] = (a[0] >> 4) & 0x0f0f0f0f;- aux[3] = (a[1] >> 4) & 0x0f0f0f0f;+#ifndef __has_builtin+ #define __has_builtin(x) 0+#endif- float sum1 = 0, sum2 = 0;- for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {- sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)- + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)- + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)- + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)- + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)- + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)- + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)- +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);- sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]- + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];+typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));+typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));+static __device__ __forceinline__ int __vsubss4(const int a, const int b) {+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);+#if __has_builtin(__builtin_elementwise_sub_sat)+ const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);+ return reinterpret_cast<const int &>(c);+#else+ int8x4_t c;+ int16_t tmp;+#pragma unroll+ for (int i = 0; i < 4; i++) {+ tmp = va[i] - vb[i];+ if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();+ if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();+ c[i] = tmp;+ }+ return reinterpret_cast<int &>(c);+#endif // __has_builtin(__builtin_elementwise_sub_sat)+}- }- tmp += dall * sum1 - dmin * sum2;+static __device__ __forceinline__ int __vsub4(const int a, const int b) {+ return __vsubss4(a, b);+}+static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);+ unsigned int c;+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ vc[i] = va[i] == vb[i] ? 0xff : 0x00;}+ return c;+}++static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {+#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx1030__)+ c = __builtin_amdgcn_sdot4(a, b, c, false);+#elif defined(RDNA3)+ c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);+#elif defined(__gfx1010__) || defined(__gfx900__)+ int tmp1;+ int tmp2;+ asm("\n \+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \+ v_add3_u32 %0, %1, %2, %0 \n \+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \+ v_add3_u32 %0, %1, %2, %0 \n \+ "+ : "+v"(c), "=&v"(tmp1), "=&v"(tmp2)+ : "v"(a), "v"(b)+ );#else- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3- const int offset = tid * K_QUANTS_PER_ITERATION;+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);+ c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];+#endif+ return c;+}+#endif // defined(GGML_USE_HIPBLAS)- uint32_t uaux[2];- const uint8_t * d = (const uint8_t *)uaux;- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+#ifdef GGML_CUDA_F16+typedef half dfloat; // dequantize float+typedef half2 dfloat2;+#else+typedef float dfloat; // dequantize float+typedef float2 dfloat2;+#endif //GGML_CUDA_F16- const float * y = yy + i * QK_K + offset;- const uint8_t * q = x[i].qs + offset;- const uint32_t * s = (const uint32_t *)x[i].scales;+static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment- uaux[0] = s[0] & 0x0f0f0f0f;- uaux[1] = (s[0] >> 4) & 0x0f0f0f0f;+ int x32 = 0;+ x32 |= x16[0] << 0;+ x32 |= x16[1] << 16;- const float2 dall = __half22float2(x[i].dm);+ return x32;+}- float sum1 = 0, sum2 = 0;- for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {- const uint8_t ql = q[l];- sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3)- + y[l+16] * d[1] * ((ql >> 2) & 3)- + y[l+32] * d[2] * ((ql >> 4) & 3)- + y[l+48] * d[3] * ((ql >> 6) & 3);- sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7];- }- tmp += dall.x * sum1 - dall.y * sum2;- }-#endif+static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+ int x32 = 0;+ x32 |= x16[0] << 0;+ x32 |= x16[1] << 16;- if (threadIdx.x == 0) {- dst[row] = tmp;- }+ return x32;}-static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {-- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;+static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+}- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+}- const block_q3_K * x = (const block_q3_K *)vx + ib0;+template<typename T>+using to_t_cuda_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int k, cudaStream_t stream);+typedef to_t_cuda_t<float> to_fp32_cuda_t;+typedef to_t_cuda_t<half> to_fp16_cuda_t;- float tmp = 0; // partial sum for thread in warp+typedef void (*dequantize_kernel_t)(const void * vx, const int ib, const int iqs, dfloat2 & v);+typedef void (*dot_kernel_k_t)(const void * __restrict__ vx, const int ib, const int iqs, const float * __restrict__ y, float & v);+typedef void (*cpy_kernel_t)(const char * cx, char * cdst);-#if QK_K == 256+typedef void (*ggml_cuda_func_t)(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);- const uint16_t kmask1 = 0x0303;- const uint16_t kmask2 = 0x0f0f;+typedef void (*ggml_cuda_op_mul_mat_t)(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream);- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1+typedef void (*ggml_cuda_op_flatten_t)(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream);- const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop- const int step = 16/K_QUANTS_PER_ITERATION;- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0....15 or 0...7-- const uint8_t m = 1 << (4*im);+typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs);+typedef void (*allocate_tiles_cuda_t)(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc);+typedef void (*load_tiles_cuda_t)(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row);+typedef float (*vec_dot_q_mul_mat_cuda_t)(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k);- const int l0 = n*in; // 0...15 or 0...14 in steps of 2- const int q_offset = 32*im + l0;- const int y_offset = 128*im + l0;+#define WARP_SIZE 32- uint16_t utmp[4];- const int8_t * s = (const int8_t *)utmp;+#define CUDA_GELU_BLOCK_SIZE 256+#define CUDA_SILU_BLOCK_SIZE 256+#define CUDA_TANH_BLOCK_SIZE 256+#define CUDA_RELU_BLOCK_SIZE 256+#define CUDA_HARDSIGMOID_BLOCK_SIZE 256+#define CUDA_HARDSWISH_BLOCK_SIZE 256+#define CUDA_SQR_BLOCK_SIZE 256+#define CUDA_CPY_BLOCK_SIZE 32+#define CUDA_SCALE_BLOCK_SIZE 256+#define CUDA_CLAMP_BLOCK_SIZE 256+#define CUDA_ROPE_BLOCK_SIZE 256+#define CUDA_SOFT_MAX_BLOCK_SIZE 1024+#define CUDA_ALIBI_BLOCK_SIZE 32+#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32+#define CUDA_QUANTIZE_BLOCK_SIZE 256+#define CUDA_DEQUANTIZE_BLOCK_SIZE 256+#define CUDA_GET_ROWS_BLOCK_SIZE 256+#define CUDA_UPSCALE_BLOCK_SIZE 256+#define CUDA_CONCAT_BLOCK_SIZE 256+#define CUDA_PAD_BLOCK_SIZE 256+#define CUDA_ARANGE_BLOCK_SIZE 256+#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256+#define CUDA_ACC_BLOCK_SIZE 256+#define CUDA_IM2COL_BLOCK_SIZE 256+#define CUDA_POOL2D_BLOCK_SIZE 256- const uint16_t s_shift = 4*im;+#define CUDA_Q8_0_NE_ALIGN 2048- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+// dmmv = dequantize_mul_mat_vec+#ifndef GGML_CUDA_DMMV_X+#define GGML_CUDA_DMMV_X 32+#endif+#ifndef GGML_CUDA_MMV_Y+#define GGML_CUDA_MMV_Y 1+#endif- const float * y = yy + i * QK_K + y_offset;- const uint8_t * q = x[i].qs + q_offset;- const uint8_t * h = x[i].hmask + l0;+#ifndef K_QUANTS_PER_ITERATION+#define K_QUANTS_PER_ITERATION 2+#else+static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUANTS_PER_ITERATION must be 1 or 2");+#endif- const uint16_t * a = (const uint16_t *)x[i].scales;- utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);- utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);- utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);- utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);+#ifndef GGML_CUDA_PEER_MAX_BATCH_SIZE+#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128+#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE- const float d = x[i].d;+#define MUL_MAT_SRC1_COL_STRIDE 128- float sum = 0;- for (int l = 0; l < n; ++l) {- sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))- + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))- + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))- + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));- sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))- + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))- + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))- + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));- }- tmp += d * sum;+[[noreturn]]+static __device__ void no_device_code(+ const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {- }+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n",+ file_name, line, function_name, arch);+ GGML_UNUSED(arch_list);#else+ printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n",+ file_name, line, function_name, arch, arch_list);+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ __trap();- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3- const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14- const int in = offset/8; // 0 or 1- const int im = offset%8; // 0...7-- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ GGML_UNUSED(no_device_code); // suppress unused function warning+}- const float * y = yy + i * QK_K + offset;- const uint8_t * q = x[i].qs + offset;- const uint8_t * s = x[i].scales;+#ifdef __CUDA_ARCH__+#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))+#else+//#define NO_DEVICE_CODE GGML_ASSERT(false && "NO_DEVICE_CODE not valid in host code.")+#define NO_DEVICE_CODE+#endif // __CUDA_ARCH__- const float dall = (float)x[i].d;+static __device__ __forceinline__ float warp_reduce_sum(float x) {+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ x += __shfl_xor_sync(0xffffffff, x, mask, 32);+ }+ return x;+}- float sum = 0;- for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {- const uint8_t hl = x[i].hmask[im+l] >> in;- const uint8_t ql = q[l];- sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4))- + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4))- + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4))- + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4));- }- tmp += sum;+static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ a.x += __shfl_xor_sync(0xffffffff, a.x, mask, 32);+ a.y += __shfl_xor_sync(0xffffffff, a.y, mask, 32);}-#endif+ return a;+}- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+#ifdef GGML_CUDA_F16+static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));+ }+ return a;+#else+ GGML_UNUSED(a);+ NO_DEVICE_CODE;+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+}+#endif // GGML_CUDA_F16- if (threadIdx.x == 0) {- dst[row] = tmp;+static __device__ __forceinline__ float warp_reduce_max(float x) {+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, mask, 32));}+ return x;}-static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {+//static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {+//#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX+//#pragma unroll+// for (int mask = 16; mask > 0; mask >>= 1) {+// x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));+// }+// return x;+//#else+// GGML_UNUSED(x);+// NO_DEVICE_CODE;+//#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX+//}- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+static __device__ __forceinline__ float op_repeat(const float a, const float b) {+ return b;+ GGML_UNUSED(a);+}- const block_q4_K * x = (const block_q4_K *)vx + ib0;+static __device__ __forceinline__ float op_add(const float a, const float b) {+ return a + b;+}-#if QK_K == 256- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;+static __device__ __forceinline__ float op_mul(const float a, const float b) {+ return a * b;+}- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1+static __device__ __forceinline__ float op_div(const float a, const float b) {+ return a / b;+}- const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>+static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,+ int ne0, int ne1, int ne2, int ne3,+ int ne10, int ne11, int ne12, int ne13,+ /*int s0, */ int s1, int s2, int s3,+ /*int s10,*/ int s11, int s12, int s13) {+ const int i0s = blockDim.x*blockIdx.x + threadIdx.x;+ const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);+ const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;+ const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;- const int il = tid/step; // 0...3- const int ir = tid - step*il; // 0...7 or 0...3- const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4+ if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {+ return;+ }- const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224- const int in = il%2;+ const int i11 = i1 % ne11;+ const int i12 = i2 % ne12;+ const int i13 = i3 % ne13;- const int l0 = n*(2*ir + in);- const int q_offset = 32*im + l0;- const int y_offset = 64*im + l0;+ const size_t i_src0 = i3*s3 + i2*s2 + i1*s1;+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;+ const size_t i_dst = i_src0;- uint16_t aux[4];- const uint8_t * sc = (const uint8_t *)aux;+ const src0_t * src0_row = src0 + i_src0;+ const src1_t * src1_row = src1 + i_src1;+ dst_t * dst_row = dst + i_dst;-#if K_QUANTS_PER_ITERATION == 2- uint32_t q32[4];- const uint8_t * q4 = (const uint8_t *)q32;-#else- uint16_t q16[4];- const uint8_t * q4 = (const uint8_t *)q16;-#endif+ for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {+ const int i10 = i0 % ne10;+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);+ }+}- float tmp = 0; // partial sum for thread in warp+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>+static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,+ int ne0, int ne1, int ne2, int ne3,+ int ne10, int ne11, int ne12, int ne13,+ /*int s0, */ int s1, int s2, int s3,+ /*int s10,*/ int s11, int s12, int s13) {- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- const float * y1 = yy + i*QK_K + y_offset;- const float * y2 = y1 + 128;+ const int i3 = i/(ne2*ne1*ne0);+ const int i2 = (i/(ne1*ne0)) % ne2;+ const int i1 = (i/ne0) % ne1;+ const int i0 = i % ne0;- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {+ return;+ }- const uint16_t * a = (const uint16_t *)x[i].scales;- aux[0] = a[im+0] & kmask1;- aux[1] = a[im+2] & kmask1;- aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);- aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);--#if K_QUANTS_PER_ITERATION == 2- const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset);- const uint32_t * q2 = q1 + 16;-- q32[0] = q1[0] & 0x0f0f0f0f;- q32[1] = q1[0] & 0xf0f0f0f0;- q32[2] = q2[0] & 0x0f0f0f0f;- q32[3] = q2[0] & 0xf0f0f0f0;+ const int i11 = i1 % ne11;+ const int i12 = i2 % ne12;+ const int i13 = i3 % ne13;- float4 s = {0.f, 0.f, 0.f, 0.f};- float smin = 0;- for (int l = 0; l < 4; ++l) {- s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+ 4];- s.z += y2[l] * q4[l+8]; s.w += y2[l+32] * q4[l+12];- smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];- }- tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;-#else- const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset);- const uint16_t * q2 = q1 + 32;+ const size_t i_src0 = i3*s3 + i2*s2 + i1*s1;+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;+ const size_t i_dst = i_src0;- q16[0] = q1[0] & 0x0f0f;- q16[1] = q1[0] & 0xf0f0;- q16[2] = q2[0] & 0x0f0f;- q16[3] = q2[0] & 0xf0f0;+ const src0_t * src0_row = src0 + i_src0;+ const src1_t * src1_row = src1 + i_src1;+ dst_t * dst_row = dst + i_dst;- float4 s = {0.f, 0.f, 0.f, 0.f};- float smin = 0;- for (int l = 0; l < 2; ++l) {- s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2];- s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6];- smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];- }- tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;-#endif+ const int i10 = i0 % ne10;+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);+}+static __global__ void acc_f32(const float * x, const float * y, float * dst, const int ne,+ const int ne10, const int ne11, const int ne12,+ const int nb1, const int nb2, int offset) {+ const int i = blockDim.x * blockIdx.x + threadIdx.x;+ if (i >= ne) {+ return;}-#else- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);-- const int step = tid * K_QUANTS_PER_ITERATION;-- uint16_t aux16[2];- const uint8_t * s = (const uint8_t *)aux16;+ int src1_idx = i - offset;+ int oz = src1_idx / nb2;+ int oy = (src1_idx - (oz * nb2)) / nb1;+ int ox = src1_idx % nb1;+ if (src1_idx >= 0 && ox < ne10 && oy < ne11 && oz < ne12) {+ dst[i] = x[i] + y[ox + oy * ne10 + oz * ne10 * ne11];+ } else {+ dst[i] = x[i];+ }+}- float tmp = 0;+static __global__ void gelu_f32(const float * x, float * dst, const int k) {+ const float GELU_COEF_A = 0.044715f;+ const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- const uint8_t * q = x[i].qs + step;- const float * y = yy + i*QK_K + step;- const uint16_t * a = (const uint16_t *)x[i].scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;- const float d = (float)x[i].dm[0];- const float m = (float)x[i].dm[1];- float sum = 0.f;- for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {- sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2])- + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2])- + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3])- + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]);- }- tmp += sum;+ if (i >= k) {+ return;}-#endif+ float xi = x[i];+ dst[i] = 0.5f*xi*(1.0f + tanhf(SQRT_2_OVER_PI*xi*(1.0f + GELU_COEF_A*xi*xi)));+}- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+static __global__ void silu_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- if (tid == 0) {- dst[row] = tmp;+ if (i >= k) {+ return;}+ dst[i] = x[i] / (1.0f + expf(-x[i]));}-static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols) {-- const int row = blockIdx.x;- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;-- const block_q5_K * x = (const block_q5_K *)vx + ib0;+static __global__ void gelu_quick_f32(const float * x, float * dst, int k) {+ const float GELU_QUICK_COEF = -1.702f;+ const int i = blockDim.x*blockIdx.x + threadIdx.x;+ if (i >= k) {+ return;+ }+ dst[i] = x[i] * (1.0f / (1.0f + expf(GELU_QUICK_COEF * x[i])));+}- float tmp = 0; // partial sum for thread in warp+static __global__ void tanh_f32(const float * x, float * dst, int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;+ if (i >= k) {+ return;+ }+ dst[i] = tanhf(x[i]);+}-#if QK_K == 256- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;+static __global__ void relu_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- const int tid = threadIdx.x/2; // 0...15- const int ix = threadIdx.x%2;+ if (i >= k) {+ return;+ }+ dst[i] = fmaxf(x[i], 0);+}- const int il = tid/4; // 0...3- const int ir = tid - 4*il;// 0...3- const int n = 2;+static __global__ void hardsigmoid_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224- const int in = il%2;+ if (i >= k) {+ return;+ }+ dst[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));+}- const int l0 = n*(2*ir + in);- const int q_offset = 32*im + l0;- const int y_offset = 64*im + l0;+static __global__ void hardswish_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- const uint8_t hm1 = 1 << (2*im);- const uint8_t hm2 = hm1 << 4;+ if (i >= k) {+ return;+ }+ dst[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f));+}- uint16_t aux[4];- const uint8_t * sc = (const uint8_t *)aux;+static __global__ void leaky_relu_f32(const float * x, float * dst, const int k, const float negative_slope) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;+ if (i >= k) {+ return;+ }+ dst[i] = fmaxf(x[i], 0) + fminf(x[i], 0.0f) * negative_slope;+}- uint16_t q16[8];- const uint8_t * q4 = (const uint8_t *)q16;+static __global__ void sqr_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- for (int i = ix; i < num_blocks_per_row; i += 2) {+ if (i >= k) {+ return;+ }+ dst[i] = x[i] * x[i];+}- const uint8_t * ql1 = x[i].qs + q_offset;- const uint8_t * qh = x[i].qh + l0;- const float * y1 = yy + i*QK_K + y_offset;- const float * y2 = y1 + 128;+template <int block_size>+static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ const int tid = threadIdx.x;- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ float2 mean_var = make_float2(0.f, 0.f);- const uint16_t * a = (const uint16_t *)x[i].scales;- aux[0] = a[im+0] & kmask1;- aux[1] = a[im+2] & kmask1;- aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);- aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);+ for (int col = tid; col < ncols; col += block_size) {+ const float xi = x[row*ncols + col];+ mean_var.x += xi;+ mean_var.y += xi * xi;+ }- float4 sum = {0.f, 0.f, 0.f, 0.f};- float smin = 0;- const uint16_t * q1 = (const uint16_t *)ql1;- const uint16_t * q2 = q1 + 32;- q16[0] = q1[0] & 0x0f0f;- q16[1] = q1[8] & 0x0f0f;- q16[2] = (q1[0] >> 4) & 0x0f0f;- q16[3] = (q1[8] >> 4) & 0x0f0f;- q16[4] = q2[0] & 0x0f0f;- q16[5] = q2[8] & 0x0f0f;- q16[6] = (q2[0] >> 4) & 0x0f0f;- q16[7] = (q2[8] >> 4) & 0x0f0f;- for (int l = 0; l < n; ++l) {- sum.x += y1[l+ 0] * (q4[l +0] + (qh[l+ 0] & (hm1 << 0) ? 16 : 0))- + y1[l+16] * (q4[l +2] + (qh[l+16] & (hm1 << 0) ? 16 : 0));- sum.y += y1[l+32] * (q4[l +4] + (qh[l+ 0] & (hm1 << 1) ? 16 : 0))- + y1[l+48] * (q4[l +6] + (qh[l+16] & (hm1 << 1) ? 16 : 0));- sum.z += y2[l+ 0] * (q4[l +8] + (qh[l+ 0] & (hm2 << 0) ? 16 : 0))- + y2[l+16] * (q4[l+10] + (qh[l+16] & (hm2 << 0) ? 16 : 0));- sum.w += y2[l+32] * (q4[l+12] + (qh[l+ 0] & (hm2 << 1) ? 16 : 0))- + y2[l+48] * (q4[l+14] + (qh[l+16] & (hm2 << 1) ? 16 : 0));- smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]- + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];+ // sum up partial sums+ mean_var = warp_reduce_sum(mean_var);+ if (block_size > WARP_SIZE) {+ __shared__ float2 s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = mean_var;}- tmp += dall * (sum.x * sc[0] + sum.y * sc[1] + sum.z * sc[4] + sum.w * sc[5]) - dmin * smin;+ __syncthreads();+ mean_var = s_sum[lane_id];+ mean_var = warp_reduce_sum(mean_var);}-#else- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);- const int step = tid * K_QUANTS_PER_ITERATION;- const int im = step/8;- const int in = step%8;+ const float mean = mean_var.x / ncols;+ const float var = mean_var.y / ncols - mean * mean;+ const float inv_std = rsqrtf(var + eps);- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- const uint8_t * q = x[i].qs + step;- const int8_t * s = x[i].scales;- const float * y = yy + i*QK_K + step;- const float d = x[i].d;- float sum = 0.f;- for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {- const uint8_t h = x[i].qh[in+j] >> im;- sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16))- + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16))- + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16))- + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16));- }- tmp += sum;+ for (int col = tid; col < ncols; col += block_size) {+ dst[row*ncols + col] = (x[row*ncols + col] - mean) * inv_std;}-#endif-- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+}- if (threadIdx.x == 0) {- dst[row] = tmp;+static __global__ void concat_f32(const float * x,const float * y, float * dst, const int ne0, const int ne02) {+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;+ if (nidx >= ne0) {+ return;+ }+ // operation+ int offset_dst =+ nidx ++ blockIdx.y * ne0 ++ blockIdx.z * ne0 * gridDim.y;+ if (blockIdx.z < ne02) { // src0+ int offset_src =+ nidx ++ blockIdx.y * ne0 ++ blockIdx.z * ne0 * gridDim.y;+ dst[offset_dst] = x[offset_src];+ } else {+ int offset_src =+ nidx ++ blockIdx.y * ne0 ++ (blockIdx.z - ne02) * ne0 * gridDim.y;+ dst[offset_dst] = y[offset_src];}}-static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {-- static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");+static __global__ void upscale_f32(const float * x, float * dst, const int ne00, const int ne00xne01, const int scale_factor) {+ // blockIdx.z: idx of ne02*ne03+ // blockIdx.y: idx of ne01*scale_factor, aka ne1+ // blockIDx.x: idx of ne00*scale_factor / BLOCK_SIZE+ // ne00xne01: ne00 * ne01+ int ne0 = ne00 * scale_factor;+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;+ if (nidx >= ne0) {+ return;+ }+ // operation+ int i00 = nidx / scale_factor;+ int i01 = blockIdx.y / scale_factor;+ int offset_src =+ i00 ++ i01 * ne00 ++ blockIdx.z * ne00xne01;+ int offset_dst =+ nidx ++ blockIdx.y * ne0 ++ blockIdx.z * ne0 * gridDim.y;+ dst[offset_dst] = x[offset_src];+}- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;+static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02, const int ne03) {+ // blockIdx.z: idx of ne2*ne3, aka ne02*ne03+ // blockIdx.y: idx of ne1+ // blockIDx.x: idx of ne0 / BLOCK_SIZE+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;+ if (nidx >= ne0) {+ return;+ }- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+ // operation+ int offset_dst =+ nidx ++ blockIdx.y * ne0 ++ blockIdx.z * ne0 * gridDim.y;+ if (nidx < ne00 && blockIdx.y < ne01 && blockIdx.z < ne02*ne03) {+ int offset_src =+ nidx ++ blockIdx.y * ne00 ++ blockIdx.z * ne00 * ne01;+ dst[offset_dst] = x[offset_src];+ } else {+ dst[offset_dst] = 0.0f;+ }+}- const block_q6_K * x = (const block_q6_K *)vx + ib0;+static __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {+ // blockIDx.x: idx of ne0 / BLOCK_SIZE+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;+ if (nidx >= ne0) {+ return;+ }+ dst[nidx] = start + step * nidx;+}-#if QK_K == 256+static __global__ void timestep_embedding_f32(const float * timesteps, float * dst, const int nb1, const int dim, const int max_period) {+ // blockIDx.y: idx of timesteps->ne[0]+ // blockIDx.x: idx of ((dim + 1) / 2) / BLOCK_SIZE+ int i = blockIdx.y;+ int j = threadIdx.x + blockIdx.x * blockDim.x;+ float * embed_data = (float *)((char *)dst + i*nb1);- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1+ if (dim % 2 != 0 && j == ((dim + 1) / 2)) {+ embed_data[dim] = 0.f;+ }- const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8+ int half = dim / 2;+ if (j >= half) {+ return;+ }- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0...15 or 0...7+ float timestep = timesteps[i];+ float freq = (float)expf(-logf(max_period) * j / half);+ float arg = timestep * freq;+ embed_data[j] = cosf(arg);+ embed_data[j + half] = sinf(arg);+}-#if K_QUANTS_PER_ITERATION == 1- const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15- const int is = 0;-#else- const int l0 = 4 * in; // 0, 4, 8, ..., 28- const int is = in / 4;-#endif- const int ql_offset = 64*im + l0;- const int qh_offset = 32*im + l0;- const int s_offset = 8*im + is;- const int y_offset = 128*im + l0;+template <int block_size>+static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {+ // blockIdx.x: num_groups idx+ // threadIdx.x: block_size idx+ int start = blockIdx.x * group_size;+ int end = start + group_size;- float tmp = 0; // partial sum for thread in warp+ start += threadIdx.x;- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+ if (end >= ne_elements) {+ end = ne_elements;+ }- const float * y = yy + i * QK_K + y_offset;- const uint8_t * ql = x[i].ql + ql_offset;- const uint8_t * qh = x[i].qh + qh_offset;- const int8_t * s = x[i].scales + s_offset;+ float tmp = 0.0f; // partial sum for thread in warp- const float d = x[i].d;+ for (int j = start; j < end; j += block_size) {+ tmp += x[j];+ }-#if K_QUANTS_PER_ITERATION == 1- float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)- + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)- + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)- + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)- + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)- + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)- + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)- +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);- tmp += sum;-#else- float sum = 0;- for (int l = 0; l < 4; ++l) {- sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)- + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)- + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)- + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);+ tmp = warp_reduce_sum(tmp);+ if (block_size > WARP_SIZE) {+ __shared__ float s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = tmp;}- tmp += sum;-#endif-+ __syncthreads();+ tmp = s_sum[lane_id];+ tmp = warp_reduce_sum(tmp);}-#else-- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...7- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0...3-- const int step = tid * K_QUANTS_PER_ITERATION;+ float mean = tmp / group_size;+ tmp = 0.0f;- float tmp = 0; // partial sum for thread in warp+ for (int j = start; j < end; j += block_size) {+ float xi = x[j] - mean;+ dst[j] = xi;+ tmp += xi * xi;+ }- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ tmp = warp_reduce_sum(tmp);+ if (block_size > WARP_SIZE) {+ __shared__ float s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = tmp;+ }+ __syncthreads();+ tmp = s_sum[lane_id];+ tmp = warp_reduce_sum(tmp);+ }- const float * y = yy + i * QK_K + step;- const uint8_t * ql = x[i].ql + step;- const uint8_t * qh = x[i].qh + step;- const int8_t * s = x[i].scales;+ float variance = tmp / group_size;+ float scale = rsqrtf(variance + eps);+ for (int j = start; j < end; j += block_size) {+ dst[j] *= scale;+ }+}- const float d = x[i+0].d;+template <int block_size>+static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ const int tid = threadIdx.x;- float sum = 0;- for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {- sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32)- + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32)- + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32)- + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32);- }- tmp += sum;+ float tmp = 0.0f; // partial sum for thread in warp+ for (int col = tid; col < ncols; col += block_size) {+ const float xi = x[row*ncols + col];+ tmp += xi * xi;}-#endif-- // sum up partial sums and write back result+ // sum up partial sumstmp = warp_reduce_sum(tmp);-- if (tid == 0) {- dst[row] = tmp;+ if (block_size > WARP_SIZE) {+ __shared__ float s_sum[32];+ int warp_id = threadIdx.x / WARP_SIZE;+ int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ s_sum[warp_id] = tmp;+ }+ __syncthreads();+ tmp = s_sum[lane_id];+ tmp = warp_reduce_sum(tmp);}-}-static __device__ void convert_f16(const void * vx, const int ib, const int iqs, dfloat2 & v){- const half * x = (const half *) vx;+ const float mean = tmp / ncols;+ const float scale = rsqrtf(mean + eps);- // automatic half -> float type cast if dfloat == float- v.x = x[ib + iqs + 0];- v.y = x[ib + iqs + 1];+ for (int col = tid; col < ncols; col += block_size) {+ dst[row*ncols + col] = scale * x[row*ncols + col];+ }}-static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int kx, const int kx_padded) {- const int ix = blockDim.x*blockIdx.x + threadIdx.x;-- if (ix >= kx_padded) {- return;- }+static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int ib, const int iqs, dfloat2 & v){+ const block_q4_0 * x = (const block_q4_0 *) vx;- const int iy = blockDim.y*blockIdx.y + threadIdx.y;+ const dfloat d = x[ib].d;- const int i_padded = iy*kx_padded + ix;+ const int vui = x[ib].qs[iqs];- block_q8_1 * y = (block_q8_1 *) vy;+ v.x = vui & 0xF;+ v.y = vui >> 4;- const int ib = i_padded / QK8_1; // block index- const int iqs = i_padded % QK8_1; // quant index-- const float xi = ix < kx ? x[iy*kx + ix] : 0.0f;- float amax = fabsf(xi);- float sum = xi;+#ifdef GGML_CUDA_F16+ v = __hsub2(v, {8.0f, 8.0f});+ v = __hmul2(v, {d, d});+#else+ v.x = (v.x - 8.0f) * d;+ v.y = (v.y - 8.0f) * d;+#endif // GGML_CUDA_F16+}- amax = warp_reduce_max(amax);- sum = warp_reduce_sum(sum);+static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int ib, const int iqs, dfloat2 & v){+ const block_q4_1 * x = (const block_q4_1 *) vx;- const float d = amax / 127;- const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);+ const dfloat d = __low2half(x[ib].dm);+ const dfloat m = __high2half(x[ib].dm);- y[ib].qs[iqs] = q;+ const int vui = x[ib].qs[iqs];- if (iqs > 0) {- return;- }+ v.x = vui & 0xF;+ v.y = vui >> 4;- reinterpret_cast<half&>(y[ib].ds.x) = d;- reinterpret_cast<half&>(y[ib].ds.y) = sum;+#ifdef GGML_CUDA_F16+ v = __hmul2(v, {d, d});+ v = __hadd2(v, {m, m});+#else+ v.x = (v.x * d) + m;+ v.y = (v.y * d) + m;+#endif // GGML_CUDA_F16}-template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static __global__ void k_get_rows(- const void * src0, const int32_t * src1, dst_t * dst,- int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/- /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/- /*size_t s0,*/ size_t s1, size_t s2, size_t s3,- /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,- size_t s10, size_t s11, size_t s12/*, size_t s13*/) {-- const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;- const int i10 = blockDim.y*blockIdx.y + threadIdx.y;- const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;- const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;-- if (i00 >= ne00) {- return;- }+static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int ib, const int iqs, dfloat2 & v){+ const block_q5_0 * x = (const block_q5_0 *) vx;- const int i01 = src1[i10*s10 + i11*s11 + i12*s12];+ const dfloat d = x[ib].d;- dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;- const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;+ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));- const int ib = i00/qk; // block index- const int iqs = (i00%qk)/qr; // quant index- const int iybs = i00 - i00%qk; // dst block start index- const int y_offset = qr == 1 ? 1 : qk/2;+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;- // dequantize- dfloat2 v;- dequantize_kernel(src0_row, ib, iqs, v);+ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);- dst_row[iybs + iqs + 0] = v.x;- dst_row[iybs + iqs + y_offset] = v.y;+#ifdef GGML_CUDA_F16+ v = __hsub2(v, {16.0f, 16.0f});+ v = __hmul2(v, {d, d});+#else+ v.x = (v.x - 16.0f) * d;+ v.y = (v.y - 16.0f) * d;+#endif // GGML_CUDA_F16}-template<typename src0_t, typename dst_t>-static __global__ void k_get_rows_float(- const src0_t * src0, const int32_t * src1, dst_t * dst,- int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/- /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/- /*size_t s0,*/ size_t s1, size_t s2, size_t s3,- /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,- size_t s10, size_t s11, size_t s12/*, size_t s13*/) {+static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int ib, const int iqs, dfloat2 & v){+ const block_q5_1 * x = (const block_q5_1 *) vx;- const int i00 = blockIdx.x*blockDim.x + threadIdx.x;- const int i10 = blockDim.y*blockIdx.y + threadIdx.y;- const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;- const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;+ const dfloat d = __low2half(x[ib].dm);+ const dfloat m = __high2half(x[ib].dm);- if (i00 >= ne00) {- return;- }+ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));- const int i01 = src1[i10*s10 + i11*s11 + i12*s12];+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;- dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;- const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);+ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);- dst_row[i00] = src0_row[i00];+#ifdef GGML_CUDA_F16+ v = __hmul2(v, {d, d});+ v = __hadd2(v, {m, m});+#else+ v.x = (v.x * d) + m;+ v.y = (v.y * d) + m;+#endif // GGML_CUDA_F16}-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int k) {- const int i = 2*(blockDim.x*blockIdx.x + threadIdx.x);-- if (i >= k) {- return;- }+static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int ib, const int iqs, dfloat2 & v){+ const block_q8_0 * x = (const block_q8_0 *) vx;- const int ib = i/qk; // block index- const int iqs = (i%qk)/qr; // quant index- const int iybs = i - i%qk; // y block start index- const int y_offset = qr == 1 ? 1 : qk/2;+ const dfloat d = x[ib].d;- // dequantize- dfloat2 v;- dequantize_kernel(vx, ib, iqs, v);+ v.x = x[ib].qs[iqs + 0];+ v.y = x[ib].qs[iqs + 1];- y[iybs + iqs + 0] = v.x;- y[iybs + iqs + y_offset] = v.y;+#ifdef GGML_CUDA_F16+ v = __hmul2(v, {d, d});+#else+ v.x *= d;+ v.y *= d;+#endif // GGML_CUDA_F16}-template <typename src_t, typename dst_t>-static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __restrict__ y, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+template<typename dst_t>+static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {- if (i >= k) {+ const int i = blockIdx.x;++ // assume 32 threads+ const int tid = threadIdx.x;+ const int il = tid/8;+ const int ir = tid%8;+ const int ib = 8*i + ir;+ if (ib >= nb32) {return;}- const src_t * x = (src_t *) vx;+ dst_t * y = yy + 256*i + 32*ir + 4*il;- y[i] = x[i];-}+ const block_q4_0 * x = (const block_q4_0 *)vx + ib;+ const float d = __half2float(x->d);+ const float dm = -8*d;-template <bool need_check>-static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int k) {-#if __CUDA_ARCH__ >= CC_PASCAL- constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;+ const uint8_t * q = x->qs + 4*il;- const int i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;- const int * x0 = ((int *) vx) + blockIdx.x * nint;- half2 * y2 = (half2 *) (y + i0);+ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = d * (q[l] & 0xF) + dm;+ y[l+16] = d * (q[l] >> 4) + dm;+ }+}- __shared__ int vals[nint];+template<typename dst_t>+static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {-#pragma unroll- for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {- if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {- break;- }+ const int i = blockIdx.x;- const int ix = ix0 + threadIdx.x;- vals[ix] = x0[ix];+ // assume 32 threads+ const int tid = threadIdx.x;+ const int il = tid/8;+ const int ir = tid%8;+ const int ib = 8*i + ir;+ if (ib >= nb32) {+ return;}-#pragma unroll- for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {- if (need_check && i0 + iy + 2*threadIdx.x >= k) {- return;- }+ dst_t * y = yy + 256*i + 32*ir + 4*il;- const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);- const half d = *b0;- const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];+ const block_q4_1 * x = (const block_q4_1 *)vx + ib;+ const float2 d = __half22float2(x->dm);- y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));+ const uint8_t * q = x->qs + 4*il;++ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = d.x * (q[l] & 0xF) + d.y;+ y[l+16] = d.x * (q[l] >> 4) + d.y;}-#else- (void) vx; (void) y; (void) k;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_PASCAL}-// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called-// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q--#define VDR_Q4_0_Q8_1_MMVQ 2-#define VDR_Q4_0_Q8_1_MMQ 4--template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(- const int * v, const int * u, const float & d4, const half2 & ds8) {+//================================== k-quants-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+template<typename dst_t>+static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {-#pragma unroll- for (int i = 0; i < vdr; ++i) {- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;+ const int i = blockIdx.x;+ const block_q2_K * x = (const block_q2_K *) vx;- // SIMD dot product of quantized values- sumi = __dp4a(vi0, u[2*i+0], sumi);- sumi = __dp4a(vi1, u[2*i+1], sumi);- }+ const int tid = threadIdx.x;+#if QK_K == 256+ const int n = tid/32;+ const int l = tid - 32*n;+ const int is = 8*n + l/16;- const float2 ds8f = __half22float2(ds8);+ const uint8_t q = x[i].qs[32*n + l];+ dst_t * y = yy + i*QK_K + 128*n;- // second part effectively subtracts 8 from each quant value- return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);+ float dall = __low2half(x[i].dm);+ float dmin = __high2half(x[i].dm);+ y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);+ y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);+ y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);+ y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ const int is = tid/16; // 0 or 1+ const int il = tid%16; // 0...15+ const uint8_t q = x[i].qs[il] >> (2*is);+ dst_t * y = yy + i*QK_K + 16*is + il;+ float dall = __low2half(x[i].dm);+ float dmin = __high2half(x[i].dm);+ y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);+ y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4);+#endif+}-#define VDR_Q4_1_Q8_1_MMVQ 2-#define VDR_Q4_1_Q8_1_MMQ 4+template<typename dst_t>+static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {-template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(- const int * v, const int * u, const half2 & dm4, const half2 & ds8) {+ const int i = blockIdx.x;+ const block_q3_K * x = (const block_q3_K *) vx;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+#if QK_K == 256+ const int r = threadIdx.x/4;+ const int tid = r/2;+ const int is0 = r%2;+ const int l0 = 16*is0 + 4*(threadIdx.x%4);+ const int n = tid / 4;+ const int j = tid - 4*n;-#pragma unroll- for (int i = 0; i < vdr; ++i) {- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;+ uint8_t m = 1 << (4*n + j);+ int is = 8*n + 2*j + is0;+ int shift = 2*j;- // SIMD dot product of quantized values- sumi = __dp4a(vi0, u[2*i+0], sumi);- sumi = __dp4a(vi1, u[2*i+1], sumi);- }+ int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :+ is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :+ is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :+ (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);+ float d_all = x[i].d;+ float dl = d_all * (us - 32);-#ifdef GGML_CUDA_F16- const float2 tmp = __half22float2(__hmul2(dm4, ds8));- const float d4d8 = tmp.x;- const float m4s8 = tmp.y;-#else- const float2 dm4f = __half22float2(dm4);- const float2 ds8f = __half22float2(ds8);- const float d4d8 = dm4f.x * ds8f.x;- const float m4s8 = dm4f.y * ds8f.y;-#endif // GGML_CUDA_F16+ dst_t * y = yy + i*QK_K + 128*n + 32*j;+ const uint8_t * q = x[i].qs + 32*n;+ const uint8_t * hm = x[i].hmask;- // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it- return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));+ for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ const int tid = threadIdx.x;+ const int is = tid/16; // 0 or 1+ const int il = tid%16; // 0...15+ const int im = il/8; // 0...1+ const int in = il%8; // 0...7-#define VDR_Q5_0_Q8_1_MMVQ 2-#define VDR_Q5_0_Q8_1_MMQ 4+ dst_t * y = yy + i*QK_K + 16*is + il;-template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(- const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {+ const uint8_t q = x[i].qs[il] >> (2*is);+ const uint8_t h = x[i].hmask[in] >> (2*is + im);+ const float d = (float)x[i].d;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+ if (is == 0) {+ y[ 0] = d * ((x[i].scales[0] & 0xF) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));+ y[32] = d * ((x[i].scales[1] & 0xF) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));+ } else {+ y[ 0] = d * ((x[i].scales[0] >> 4) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));+ y[32] = d * ((x[i].scales[1] >> 4) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));+ }+#endif-#pragma unroll- for (int i = 0; i < vdr; ++i) {- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28- sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values+}- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28- sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values+#if QK_K == 256+static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {+ if (j < 4) {+ d = q[j] & 63; m = q[j + 4] & 63;+ } else {+ d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);+ m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);}+}+#endif- const float2 ds8f = __half22float2(ds8);+template<typename dst_t>+static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const block_q4_K * x = (const block_q4_K *) vx;- // second part effectively subtracts 16 from each quant value- return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ const int i = blockIdx.x;-#define VDR_Q5_1_Q8_1_MMVQ 2-#define VDR_Q5_1_Q8_1_MMQ 4+#if QK_K == 256+ // assume 32 threads+ const int tid = threadIdx.x;+ const int il = tid/8;+ const int ir = tid%8;+ const int is = 2*il;+ const int n = 4;-template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(- const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {+ dst_t * y = yy + i*QK_K + 64*il + n*ir;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);-#pragma unroll- for (int i = 0; i < vdr; ++i) {- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28- sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values+ const uint8_t * q = x[i].qs + 32*il + n*ir;- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28- sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values+ uint8_t sc, m;+ get_scale_min_k4(is + 0, x[i].scales, sc, m);+ const float d1 = dall * sc; const float m1 = dmin * m;+ get_scale_min_k4(is + 1, x[i].scales, sc, m);+ const float d2 = dall * sc; const float m2 = dmin * m;+ for (int l = 0; l < n; ++l) {+ y[l + 0] = d1 * (q[l] & 0xF) - m1;+ y[l +32] = d2 * (q[l] >> 4) - m2;}--#ifdef GGML_CUDA_F16- const float2 tmp = __half22float2(__hmul2(dm5, ds8));- const float d5d8 = tmp.x;- const float m5s8 = tmp.y;#else- const float2 dm5f = __half22float2(dm5);- const float2 ds8f = __half22float2(ds8);- const float d5d8 = dm5f.x * ds8f.x;- const float m5s8 = dm5f.y * ds8f.y;-#endif // GGML_CUDA_F16+ const int tid = threadIdx.x;+ const uint8_t * q = x[i].qs;+ dst_t * y = yy + i*QK_K;+ const float d = (float)x[i].dm[0];+ const float m = (float)x[i].dm[1];+ y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4);+ y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4);+#endif+}- // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it- return sumi*d5d8 + m5s8 / (QI5_1 / vdr);+template<typename dst_t>+static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const block_q5_K * x = (const block_q5_K *) vx;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ const int i = blockIdx.x;-#define VDR_Q8_0_Q8_1_MMVQ 2-#define VDR_Q8_0_Q8_1_MMQ 8+#if QK_K == 256+ // assume 64 threads - this is very slightly better than the one below+ const int tid = threadIdx.x;+ const int il = tid/16; // il is in 0...3+ const int ir = tid%16; // ir is in 0...15+ const int is = 2*il; // is is in 0...6-template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(- const int * v, const int * u, const float & d8_0, const float & d8_1) {+ dst_t * y = yy + i*QK_K + 64*il + 2*ir;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);-#pragma unroll- for (int i = 0; i < vdr; ++i) {- // SIMD dot product of quantized values- sumi = __dp4a(v[i], u[i], sumi);- }+ const uint8_t * ql = x[i].qs + 32*il + 2*ir;+ const uint8_t * qh = x[i].qh + 2*ir;- return d8_0*d8_1 * sumi;+ uint8_t sc, m;+ get_scale_min_k4(is + 0, x[i].scales, sc, m);+ const float d1 = dall * sc; const float m1 = dmin * m;+ get_scale_min_k4(is + 1, x[i].scales, sc, m);+ const float d2 = dall * sc; const float m2 = dmin * m;++ uint8_t hm = 1 << (2*il);+ y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;+ y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;+ hm <<= 1;+ y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;+ y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ const int tid = threadIdx.x;+ const uint8_t q = x[i].qs[tid];+ const int im = tid/8; // 0...3+ const int in = tid%8; // 0...7+ const int is = tid/16; // 0 or 1+ const uint8_t h = x[i].qh[in] >> im;+ const float d = x[i].d;+ dst_t * y = yy + i*QK_K + tid;+ y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16));+ y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16));+#endif}-template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(- const int * v, const int * u, const half2 & dm8, const half2 & ds8) {--#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+template<typename dst_t>+static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const block_q6_K * x = (const block_q6_K *) vx;-#pragma unroll- for (int i = 0; i < vdr; ++i) {- // SIMD dot product of quantized values- sumi = __dp4a(v[i], u[i], sumi);- }+ const int i = blockIdx.x;+#if QK_K == 256-#ifdef GGML_CUDA_F16- const float2 tmp = __half22float2(__hmul2(dm8, ds8));- const float d8d8 = tmp.x;- const float m8s8 = tmp.y;-#else- const float2 dm8f = __half22float2(dm8);- const float2 ds8f = __half22float2(ds8);- const float d8d8 = dm8f.x * ds8f.x;- const float m8s8 = dm8f.y * ds8f.y;-#endif // GGML_CUDA_F16+ // assume 64 threads - this is very slightly better than the one below+ const int tid = threadIdx.x;+ const int ip = tid/32; // ip is 0 or 1+ const int il = tid - 32*ip; // 0...32+ const int is = 8*ip + il/16;- // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it- return sumi*d8d8 + m8s8 / (QI8_1 / vdr);-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ dst_t * y = yy + i*QK_K + 128*ip + il;-#define VDR_Q2_K_Q8_1_MMVQ 1-#define VDR_Q2_K_Q8_1_MMQ 2+ const float d = x[i].d;-// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(- const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,- const half2 & dm2, const float * __restrict__ d8) {+ const uint8_t * ql = x[i].ql + 64*ip + il;+ const uint8_t qh = x[i].qh[32*ip + il];+ const int8_t * sc = x[i].scales + is;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+ y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);+ y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);+ y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);+ y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);+#else-#pragma unroll- for (int i = 0; i < QR2_K; ++i) {- const int sc = scales[2*i];+ // assume 32 threads+ const int tid = threadIdx.x;+ const int ip = tid/16; // 0 or 1+ const int il = tid - 16*ip; // 0...15- const int vi = (v >> (2*i)) & 0x03030303;+ dst_t * y = yy + i*QK_K + 16*ip + il;- sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product+ const float d = x[i].d;- // fill int with 4x m- int m = sc >> 4;- m |= m << 8;- m |= m << 16;- sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values- }+ const uint8_t ql = x[i].ql[16*ip + il];+ const uint8_t qh = x[i].qh[il] >> (2*ip);+ const int8_t * sc = x[i].scales;- const float2 dm2f = __half22float2(dm2);+ y[ 0] = d * sc[ip+0] * ((int8_t)((ql & 0xF) | (((qh >> 0) & 3) << 4)) - 32);+ y[32] = d * sc[ip+2] * ((int8_t)((ql >> 4) | (((qh >> 4) & 3) << 4)) - 32);+#endif+}- return dm2f.x*sumf_d - dm2f.y*sumf_m;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+inline bool ggml_cuda_supports_mmq(enum ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ return true;+ default:+ return false;+ }}-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,- const half2 & dm2, const float & d8) {+template<typename dst_t>+static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi_d = 0;- int sumi_m = 0;+ const int i = blockIdx.x;+ const block_iq2_xxs * x = (const block_iq2_xxs *) vx;-#pragma unroll- for (int i0 = 0; i0 < QI8_1; i0 += QI8_1/2) {- int sumi_d_sc = 0;+ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint16_t * q2 = x[i].qs + 4*ib;+ const uint8_t * aux8 = (const uint8_t *)q2;+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);+ const uint32_t aux32 = q2[2] | (q2[3] << 16);+ const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+#else+ assert(false);+#endif- const int sc = scales[i0 / (QI8_1/2)];+}- // fill int with 4x m- int m = sc >> 4;- m |= m << 8;- m |= m << 16;+template<typename dst_t>+static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {-#pragma unroll- for (int i = i0; i < i0 + QI8_1/2; ++i) {- sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product- sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m- }+ const int i = blockIdx.x;+ const block_iq2_xs * x = (const block_iq2_xs *) vx;- sumi_d += sumi_d_sc * (sc & 0xF);- }+ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint16_t * q2 = x[i].qs + 4*ib;+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));+ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;+ const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+#else+ assert(false);+#endif- const float2 dm2f = __half22float2(dm2);+}- return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);+template<typename dst_t>+static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {++ const int i = blockIdx.x;+ const block_iq2_s * x = (const block_iq2_s *) vx;++ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));+ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;+ const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ assert(false);+#endif+}-#define VDR_Q3_K_Q8_1_MMVQ 1-#define VDR_Q3_K_Q8_1_MMQ 2+template<typename dst_t>+static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {-// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(- const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,- const int & scale_offset, const float & d3, const float * __restrict__ d8) {+ const int i = blockIdx.x;+ const block_iq3_xxs * x = (const block_iq3_xxs *) vx;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf = 0.0f;+ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint8_t * q3 = x[i].qs + 8*ib;+ const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);+ const uint32_t aux32 = gas[0] | (gas[1] << 16);+ const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];+ for (int j = 0; j < 4; ++j) {+ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+#else+ assert(false);+#endif-#pragma unroll- for (int i = 0; i < QR3_K; ++i) {- const int isc = scale_offset + 2*i;+}- const int isc_low = isc % (QK_K/32);- const int sc_shift_low = 4 * (isc / (QK_K/32));- const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;+template<typename dst_t>+static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int isc_high = isc % (QK_K/64);- const int sc_shift_high = 2 * (isc / (QK_K/64));- const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;+ const int i = blockIdx.x;+ const block_iq3_s * x = (const block_iq3_s *) vx;- const int sc = (sc_low | sc_high) - 32;+ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint8_t * qs = x[i].qs + 8*ib;+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));+ const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf));+ const uint8_t signs = x[i].signs[4*ib + il];+ for (int j = 0; j < 4; ++j) {+ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);+ }+#else+ assert(false);+#endif- const int vil = (vl >> (2*i)) & 0x03030303;+}- const int vih = ((vh >> i) << 2) & 0x04040404;+template<typename dst_t>+static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int vi = __vsubss4(vil, vih);+ const int i = blockIdx.x;+ const block_iq1_s * x = (const block_iq1_s *) vx;- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product+ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;+ const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;+ grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;+ grid32[0] &= 0x0f0f0f0f;+ for (int j = 0; j < 8; ++j) {+ y[j] = d * (q[j] + delta);}-- return d3 * sumf;#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ assert(false);+#endif+}-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,- const float & d3, const float & d8) {+static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+template<typename dst_t>+static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {-#pragma unroll- for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {- int sumi_sc = 0;-- for (int i = i0; i < i0 + QI8_1/2; ++i) {- sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product- }+ const int i = blockIdx.x;+ const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);- sumi += sumi_sc * scales[i0 / (QI8_1/2)];+ const int tid = threadIdx.x;+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;+ const uint8_t * q4 = x[ib].qs + 4*il;+ const float d = (float)x[ib].d;+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];+ y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];}- return d3*d8 * sumi;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define VDR_Q4_K_Q8_1_MMVQ 2-#define VDR_Q4_K_Q8_1_MMQ 8--// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {--#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;--#pragma unroll- for (int i = 0; i < QR4_K; ++i) {- const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;- const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;-- const int dot1 = __dp4a(v1i, u[2*i+1], __dp4a(v0i, u[2*i+0], 0)); // SIMD dot product- const int dot2 = __dp4a(0x01010101, u[2*i+1], __dp4a(0x01010101, u[2*i+0], 0)); // sum of u+#if QK_K != 64+template<typename dst_t>+static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int i = blockIdx.x;+ const block_iq4_xs * x = (const block_iq4_xs *)vx;- sumf_d += d8[i] * (dot1 * sc[i]);- sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values+ const int tid = threadIdx.x;+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;+ const uint8_t * q4 = x[i].qs + 16*ib + 4*il;+ const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];+ y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];}-- const float2 dm4f = __half22float2(dm4);-- return dm4f.x*sumf_d - dm4f.y*sumf_m;--#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}+#endif-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {+static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");-#pragma unroll- for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {- int sumi_d = 0;+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;-#pragma unroll- for (int j = 0; j < QI8_1; ++j) {- sumi_d = __dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product- }+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- const float2 ds8f = __half22float2(ds8[i]);+ const block_q2_K * x = (const block_q2_K *)vx + ib0;- sumf_d += ds8f.x * (sc[i] * sumi_d);- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val- }+ float tmp = 0; // partial sum for thread in warp- const float2 dm4f = __half22float2(dm4);+#if QK_K == 256+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...15+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1- return dm4f.x*sumf_d - dm4f.y*sumf_m;+ const int step = 16/K_QUANTS_PER_ITERATION;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...+ const int in = tid - step*im; // 0...15 or 0...7-#define VDR_Q5_K_Q8_1_MMVQ 2-#define VDR_Q5_K_Q8_1_MMQ 8+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2+ const int q_offset = 32*im + l0;+ const int s_offset = 8*im;+ const int y_offset = 128*im + l0;-// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(- const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {+ uint32_t aux[4];+ const uint8_t * d = (const uint8_t *)aux;+ const uint8_t * m = (const uint8_t *)(aux + 2);-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {-#pragma unroll- for (int i = 0; i < QR5_K; ++i) {- const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;- const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;+ const float * y = yy + i * QK_K + y_offset;+ const uint8_t * q = x[i].qs + q_offset;- const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;- const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);- const int v0i = vl0i | vh0i;- const int v1i = vl1i | vh1i;+ const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset);+ aux[0] = a[0] & 0x0f0f0f0f;+ aux[1] = a[1] & 0x0f0f0f0f;+ aux[2] = (a[0] >> 4) & 0x0f0f0f0f;+ aux[3] = (a[1] >> 4) & 0x0f0f0f0f;- const int dot1 = __dp4a(v0i, u[2*i+0], __dp4a(v1i, u[2*i+1], 0)); // SIMD dot product- const int dot2 = __dp4a(0x01010101, u[2*i+0], __dp4a(0x01010101, u[2*i+1], 0)); // sum of u+ float sum1 = 0, sum2 = 0;+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {+ sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)+ + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)+ + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)+ + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)+ + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)+ + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)+ + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)+ +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);+ sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]+ + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];- sumf_d += d8[i] * (dot1 * sc[i]);- sumf_m += d8[i] * (dot2 * m[i]);+ }+ tmp += dall * sum1 - dmin * sum2;}+#else+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3+ const int offset = tid * K_QUANTS_PER_ITERATION;- const float2 dm5f = __half22float2(dm5);-- return dm5f.x*sumf_d - dm5f.y*sumf_m;+ uint32_t uaux[2];+ const uint8_t * d = (const uint8_t *)uaux;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {+ const float * y = yy + i * QK_K + offset;+ const uint8_t * q = x[i].qs + offset;+ const uint32_t * s = (const uint32_t *)x[i].scales;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+ uaux[0] = s[0] & 0x0f0f0f0f;+ uaux[1] = (s[0] >> 4) & 0x0f0f0f0f;-#pragma unroll- for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {- int sumi_d = 0;+ const float2 dall = __half22float2(x[i].dm);-#pragma unroll- for (int j = 0; j < QI8_1; ++j) {- sumi_d = __dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product+ float sum1 = 0, sum2 = 0;+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {+ const uint8_t ql = q[l];+ sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3)+ + y[l+16] * d[1] * ((ql >> 2) & 3)+ + y[l+32] * d[2] * ((ql >> 4) & 3)+ + y[l+48] * d[3] * ((ql >> 6) & 3);+ sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7];}+ tmp += dall.x * sum1 - dall.y * sum2;+ }+#endif- const float2 ds8f = __half22float2(ds8[i]);+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);- sumf_d += ds8f.x * (sc[i] * sumi_d);- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val+ if (threadIdx.x == 0) {+ dst[row] = tmp;}+}- const float2 dm4f = __half22float2(dm4);+static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {- return dm4f.x*sumf_d - dm4f.y*sumf_m;+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;-#define VDR_Q6_K_Q8_1_MMVQ 1-#define VDR_Q6_K_Q8_1_MMQ 8+ const block_q3_K * x = (const block_q3_K *)vx + ib0;-// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(- const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,- const float & d, const float * __restrict__ d8) {+ float tmp = 0; // partial sum for thread in warp-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf = 0.0f;+#if QK_K == 256-#pragma unroll- for (int i = 0; i < QR6_K; ++i) {- const int sc = scales[4*i];+ const uint16_t kmask1 = 0x0303;+ const uint16_t kmask2 = 0x0f0f;- const int vil = (vl >> (4*i)) & 0x0F0F0F0F;+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1- const int vih = ((vh >> (4*i)) << 4) & 0x30303030;+ const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop+ const int step = 16/K_QUANTS_PER_ITERATION;+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...+ const int in = tid - step*im; // 0....15 or 0...7- const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32+ const uint8_t m = 1 << (4*im);- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product- }+ const int l0 = n*in; // 0...15 or 0...14 in steps of 2+ const int q_offset = 32*im + l0;+ const int y_offset = 128*im + l0;- return d*sumf;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ uint16_t utmp[4];+ const int8_t * s = (const int8_t *)utmp;-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,- const float & d6, const float * __restrict__ d8) {+ const uint16_t s_shift = 4*im;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {-#pragma unroll- for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {- int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale+ const float * y = yy + i * QK_K + y_offset;+ const uint8_t * q = x[i].qs + q_offset;+ const uint8_t * h = x[i].hmask + l0;-#pragma unroll- for (int i = i0; i < i0 + 2; ++i) {- sumi_d.x = __dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product- sumi_d.x = __dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product+ const uint16_t * a = (const uint16_t *)x[i].scales;+ utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);+ utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);+ utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);+ utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);- sumi_d.y = __dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product- sumi_d.y = __dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product+ const float d = x[i].d;++ float sum = 0;+ for (int l = 0; l < n; ++l) {+ sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))+ + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))+ + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))+ + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));+ sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))+ + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))+ + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))+ + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));}+ tmp += d * sum;- sumf_d += d8[i0/4] * (sc[i0/2+0]*sumi_d.x + sc[i0/2+1]*sumi_d.y);}-- return d6 * sumf_d;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}-static __device__ __forceinline__ float vec_dot_q4_0_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3+ const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14+ const int in = offset/8; // 0 or 1+ const int im = offset%8; // 0...7- const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- int v[VDR_Q4_0_Q8_1_MMVQ];- int u[2*VDR_Q4_0_Q8_1_MMVQ];+ const float * y = yy + i * QK_K + offset;+ const uint8_t * q = x[i].qs + offset;+ const uint8_t * s = x[i].scales;-#pragma unroll- for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);+ const float dall = (float)x[i].d;++ float sum = 0;+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {+ const uint8_t hl = x[i].hmask[im+l] >> in;+ const uint8_t ql = q[l];+ sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4))+ + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4))+ + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4))+ + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4));+ }+ tmp += sum;}+#endif- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);++ if (threadIdx.x == 0) {+ dst[row] = tmp;+ }}-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh; (void)x_sc;+static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];- __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0];+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- *x_ql = tile_x_qs;- *x_dm = (half2 *) tile_x_d;-}+ const block_q4_K * x = (const block_q4_K *)vx + ib0;-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh; (void)x_sc;- GGML_CUDA_ASSUME(i_offset >= 0);- GGML_CUDA_ASSUME(i_offset < nwarps);- GGML_CUDA_ASSUME(k >= 0);- GGML_CUDA_ASSUME(k < WARP_SIZE);+#if QK_K == 256+ const uint16_t kmask1 = 0x3f3f;+ const uint16_t kmask2 = 0x0f0f;+ const uint16_t kmask3 = 0xc0c0;- const int kbx = k / QI4_0;- const int kqsx = k % QI4_0;+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1- const block_q4_0 * bx0 = (const block_q4_0 *) vx;+ const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4- float * x_dmf = (float *) x_dm;+ const int il = tid/step; // 0...3+ const int ir = tid - step*il; // 0...7 or 0...3+ const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;+ const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224+ const int in = il%2;- if (need_check) {- i = min(i, i_max);- }+ const int l0 = n*(2*ir + in);+ const int q_offset = 32*im + l0;+ const int y_offset = 64*im + l0;- const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbx;+ uint16_t aux[4];+ const uint8_t * sc = (const uint8_t *)aux;- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);- // x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbx] = bxi->d;- }+#if K_QUANTS_PER_ITERATION == 2+ uint32_t q32[4];+ const uint8_t * q4 = (const uint8_t *)q32;+#else+ uint16_t q16[4];+ const uint8_t * q4 = (const uint8_t *)q16;+#endif- const int blocks_per_tile_x_row = WARP_SIZE / QI4_0;- const int kbxd = k % blocks_per_tile_x_row;+ float tmp = 0; // partial sum for thread in warp-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {- int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row;+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {- if (need_check) {- i = min(i, i_max);- }+ const float * y1 = yy + i*QK_K + y_offset;+ const float * y2 = y1 + 128;- const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbxd;+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);- x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbxd] = bxi->d;- }-}+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux[0] = a[im+0] & kmask1;+ aux[1] = a[im+2] & kmask1;+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);-static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh; (void)x_sc;+#if K_QUANTS_PER_ITERATION == 2+ const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset);+ const uint32_t * q2 = q1 + 16;- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));- const float * x_dmf = (const float *) x_dm;+ q32[0] = q1[0] & 0x0f0f0f0f;+ q32[1] = q1[0] & 0xf0f0f0f0;+ q32[2] = q2[0] & 0x0f0f0f0f;+ q32[3] = q2[0] & 0xf0f0f0f0;- int u[2*VDR_Q4_0_Q8_1_MMQ];+ float4 s = {0.f, 0.f, 0.f, 0.f};+ float smin = 0;+ for (int l = 0; l < 4; ++l) {+ s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+ 4];+ s.z += y2[l] * q4[l+8]; s.w += y2[l+32] * q4[l+12];+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];+ }+ tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;+#else+ const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset);+ const uint16_t * q2 = q1 + 32;-#pragma unroll- for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_0) % WARP_SIZE];- }+ q16[0] = q1[0] & 0x0f0f;+ q16[1] = q1[0] & 0xf0f0;+ q16[2] = q2[0] & 0x0f0f;+ q16[3] = q2[0] & 0xf0f0;- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>- (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dmf[i * (WARP_SIZE/QI4_0) + i/QI4_0 + k/QI4_0],- y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);-}+ float4 s = {0.f, 0.f, 0.f, 0.f};+ float smin = 0;+ for (int l = 0; l < 2; ++l) {+ s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2];+ s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6];+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];+ }+ tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;+#endif-static __device__ __forceinline__ float vec_dot_q4_1_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ }+#else+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);- const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;+ const int step = tid * K_QUANTS_PER_ITERATION;- int v[VDR_Q4_1_Q8_1_MMVQ];- int u[2*VDR_Q4_1_Q8_1_MMVQ];+ uint16_t aux16[2];+ const uint8_t * s = (const uint8_t *)aux16;-#pragma unroll- for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);+ float tmp = 0;++ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ const uint8_t * q = x[i].qs + step;+ const float * y = yy + i*QK_K + step;+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;+ const float d = (float)x[i].dm[0];+ const float m = (float)x[i].dm[1];+ float sum = 0.f;+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {+ sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2])+ + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2])+ + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3])+ + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]);+ }+ tmp += sum;}- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);+#endif++ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);++ if (tid == 0) {+ dst[row] = tmp;+ }}-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh; (void)x_sc;+static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols) {- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1];+ const int row = blockIdx.x;+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- *x_ql = tile_x_qs;- *x_dm = tile_x_dm;-}+ const block_q5_K * x = (const block_q5_K *)vx + ib0;-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh; (void)x_sc;+ float tmp = 0; // partial sum for thread in warp- GGML_CUDA_ASSUME(i_offset >= 0);- GGML_CUDA_ASSUME(i_offset < nwarps);- GGML_CUDA_ASSUME(k >= 0);- GGML_CUDA_ASSUME(k < WARP_SIZE);+#if QK_K == 256+ const uint16_t kmask1 = 0x3f3f;+ const uint16_t kmask2 = 0x0f0f;+ const uint16_t kmask3 = 0xc0c0;- const int kbx = k / QI4_1;- const int kqsx = k % QI4_1;+ const int tid = threadIdx.x/2; // 0...15+ const int ix = threadIdx.x%2;- const block_q4_1 * bx0 = (const block_q4_1 *) vx;+ const int il = tid/4; // 0...3+ const int ir = tid - 4*il;// 0...3+ const int n = 2;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;+ const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224+ const int in = il%2;- if (need_check) {- i = min(i, i_max);- }+ const int l0 = n*(2*ir + in);+ const int q_offset = 32*im + l0;+ const int y_offset = 64*im + l0;- const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbx;+ const uint8_t hm1 = 1 << (2*im);+ const uint8_t hm2 = hm1 << 4;- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);- }+ uint16_t aux[4];+ const uint8_t * sc = (const uint8_t *)aux;- const int blocks_per_tile_x_row = WARP_SIZE / QI4_1;- const int kbxd = k % blocks_per_tile_x_row;+ uint16_t q16[8];+ const uint8_t * q4 = (const uint8_t *)q16;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {- int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row;+ for (int i = ix; i < num_blocks_per_row; i += 2) {- if (need_check) {- i = min(i, i_max);- }+ const uint8_t * ql1 = x[i].qs + q_offset;+ const uint8_t * qh = x[i].qh + l0;+ const float * y1 = yy + i*QK_K + y_offset;+ const float * y2 = y1 + 128;- const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbxd;+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);- x_dm[i * (WARP_SIZE/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux[0] = a[im+0] & kmask1;+ aux[1] = a[im+2] & kmask1;+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);++ float4 sum = {0.f, 0.f, 0.f, 0.f};+ float smin = 0;+ const uint16_t * q1 = (const uint16_t *)ql1;+ const uint16_t * q2 = q1 + 32;+ q16[0] = q1[0] & 0x0f0f;+ q16[1] = q1[8] & 0x0f0f;+ q16[2] = (q1[0] >> 4) & 0x0f0f;+ q16[3] = (q1[8] >> 4) & 0x0f0f;+ q16[4] = q2[0] & 0x0f0f;+ q16[5] = q2[8] & 0x0f0f;+ q16[6] = (q2[0] >> 4) & 0x0f0f;+ q16[7] = (q2[8] >> 4) & 0x0f0f;+ for (int l = 0; l < n; ++l) {+ sum.x += y1[l+ 0] * (q4[l +0] + (qh[l+ 0] & (hm1 << 0) ? 16 : 0))+ + y1[l+16] * (q4[l +2] + (qh[l+16] & (hm1 << 0) ? 16 : 0));+ sum.y += y1[l+32] * (q4[l +4] + (qh[l+ 0] & (hm1 << 1) ? 16 : 0))+ + y1[l+48] * (q4[l +6] + (qh[l+16] & (hm1 << 1) ? 16 : 0));+ sum.z += y2[l+ 0] * (q4[l +8] + (qh[l+ 0] & (hm2 << 0) ? 16 : 0))+ + y2[l+16] * (q4[l+10] + (qh[l+16] & (hm2 << 0) ? 16 : 0));+ sum.w += y2[l+32] * (q4[l+12] + (qh[l+ 0] & (hm2 << 1) ? 16 : 0))+ + y2[l+48] * (q4[l+14] + (qh[l+16] & (hm2 << 1) ? 16 : 0));+ smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]+ + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];+ }+ tmp += dall * (sum.x * sc[0] + sum.y * sc[1] + sum.z * sc[4] + sum.w * sc[5]) - dmin * smin;}-}-static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh; (void)x_sc;+#else+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);+ const int step = tid * K_QUANTS_PER_ITERATION;+ const int im = step/8;+ const int in = step%8;- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ const uint8_t * q = x[i].qs + step;+ const int8_t * s = x[i].scales;+ const float * y = yy + i*QK_K + step;+ const float d = x[i].d;+ float sum = 0.f;+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {+ const uint8_t h = x[i].qh[in+j] >> im;+ sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16))+ + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16))+ + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16))+ + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16));+ }+ tmp += sum;+ }+#endif- int u[2*VDR_Q4_1_Q8_1_MMQ];+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);-#pragma unroll- for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_1) % WARP_SIZE];+ if (threadIdx.x == 0) {+ dst[row] = tmp;}-- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>- (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dm[i * (WARP_SIZE/QI4_1) + i/QI4_1 + k/QI4_1],- y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);}-static __device__ __forceinline__ float vec_dot_q5_0_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {- const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");- int vl[VDR_Q5_0_Q8_1_MMVQ];- int vh[VDR_Q5_0_Q8_1_MMVQ];- int u[2*VDR_Q5_0_Q8_1_MMVQ];+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;-#pragma unroll- for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {- vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);- vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);- }+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);-}+ const block_q6_K * x = (const block_q6_K *)vx + ib0;-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh; (void)x_sc;+#if QK_K == 256- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];- __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0];+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1- *x_ql = tile_x_ql;- *x_dm = (half2 *) tile_x_d;-}+ const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh; (void)x_sc;+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...+ const int in = tid - step*im; // 0...15 or 0...7- GGML_CUDA_ASSUME(i_offset >= 0);- GGML_CUDA_ASSUME(i_offset < nwarps);- GGML_CUDA_ASSUME(k >= 0);- GGML_CUDA_ASSUME(k < WARP_SIZE);+#if K_QUANTS_PER_ITERATION == 1+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15+ const int is = 0;+#else+ const int l0 = 4 * in; // 0, 4, 8, ..., 28+ const int is = in / 4;+#endif+ const int ql_offset = 64*im + l0;+ const int qh_offset = 32*im + l0;+ const int s_offset = 8*im + is;+ const int y_offset = 128*im + l0;- const int kbx = k / QI5_0;- const int kqsx = k % QI5_0;+ float tmp = 0; // partial sum for thread in warp- const block_q5_0 * bx0 = (const block_q5_0 *) vx;+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;+ const float * y = yy + i * QK_K + y_offset;+ const uint8_t * ql = x[i].ql + ql_offset;+ const uint8_t * qh = x[i].qh + qh_offset;+ const int8_t * s = x[i].scales + s_offset;- if (need_check) {- i = min(i, i_max);+ const float d = x[i].d;++#if K_QUANTS_PER_ITERATION == 1+ float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)+ + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)+ + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)+ + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)+ + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)+ + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)+ + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)+ +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);+ tmp += sum;+#else+ float sum = 0;+ for (int l = 0; l < 4; ++l) {+ sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)+ + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)+ + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)+ + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);}+ tmp += sum;+#endif- const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbx;+ }- const int ql = get_int_from_uint8(bxi->qs, kqsx);- const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0));+#else- int qs0 = (ql >> 0) & 0x0F0F0F0F;- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28- qs0 = __vsubss4(qs0, 0x10101010); // subtract 16+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...7+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0...3- x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;+ const int step = tid * K_QUANTS_PER_ITERATION;- int qs1 = (ql >> 4) & 0x0F0F0F0F;- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28- qs1 = __vsubss4(qs1, 0x10101010); // subtract 16+ float tmp = 0; // partial sum for thread in warp- x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;- }+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- const int blocks_per_tile_x_row = WARP_SIZE / QI5_0;- const int kbxd = k % blocks_per_tile_x_row;- float * x_dmf = (float *) x_dm;+ const float * y = yy + i * QK_K + step;+ const uint8_t * ql = x[i].ql + step;+ const uint8_t * qh = x[i].qh + step;+ const int8_t * s = x[i].scales;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {- int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row;+ const float d = x[i+0].d;- if (need_check) {- i = min(i, i_max);+ float sum = 0;+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {+ sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32)+ + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32)+ + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32)+ + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32);}+ tmp += sum;- const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbxd;+ }- x_dmf[i * (WARP_SIZE/QI5_0) + i / QI5_0 + kbxd] = bxi->d;+#endif++ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);++ if (tid == 0) {+ dst[row] = tmp;}}-static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh; (void)x_sc;+static __device__ void convert_f16(const void * vx, const int ib, const int iqs, dfloat2 & v){+ const half * x = (const half *) vx;- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));- const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0;- const float * x_dmf = (const float *) x_dm;- const float * y_df = (const float *) y_ds;+ // automatic half -> float type cast if dfloat == float+ v.x = x[ib + iqs + 0];+ v.y = x[ib + iqs + 1];+}- int u[2*VDR_Q5_0_Q8_1_MMQ];+static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int kx, const int kx_padded) {+ const int ix = blockDim.x*blockIdx.x + threadIdx.x;-#pragma unroll- for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_0) % WARP_SIZE];+ if (ix >= kx_padded) {+ return;}- return vec_dot_q8_0_q8_1_impl<QR5_0*VDR_Q5_0_Q8_1_MMQ>- (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);-}+ const int iy = blockDim.y*blockIdx.y + threadIdx.y;-static __device__ __forceinline__ float vec_dot_q5_1_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ const int i_padded = iy*kx_padded + ix;- const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;+ block_q8_1 * y = (block_q8_1 *) vy;- int vl[VDR_Q5_1_Q8_1_MMVQ];- int vh[VDR_Q5_1_Q8_1_MMVQ];- int u[2*VDR_Q5_1_Q8_1_MMVQ];+ const int ib = i_padded / QK8_1; // block index+ const int iqs = i_padded % QK8_1; // quant index-#pragma unroll- for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {- vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);- vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);- }+ const float xi = ix < kx ? x[iy*kx + ix] : 0.0f;+ float amax = fabsf(xi);+ float sum = xi;- return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);-}+ amax = warp_reduce_max(amax);+ sum = warp_reduce_sum(sum);-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh; (void)x_sc;+ const float d = amax / 127;+ const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1];+ y[ib].qs[iqs] = q;- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;-}+ if (iqs > 0) {+ return;+ }-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh; (void)x_sc;+ reinterpret_cast<half&>(y[ib].ds.x) = d;+ reinterpret_cast<half&>(y[ib].ds.y) = sum;+}- GGML_CUDA_ASSUME(i_offset >= 0);- GGML_CUDA_ASSUME(i_offset < nwarps);- GGML_CUDA_ASSUME(k >= 0);- GGML_CUDA_ASSUME(k < WARP_SIZE);+template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static __global__ void k_get_rows(+ const void * src0, const int32_t * src1, dst_t * dst,+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {- const int kbx = k / QI5_1;- const int kqsx = k % QI5_1;+ const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;- const block_q5_1 * bx0 = (const block_q5_1 *) vx;+ if (i00 >= ne00) {+ return;+ }-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];- if (need_check) {- i = min(i, i_max);- }+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;- const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbx;+ const int ib = i00/qk; // block index+ const int iqs = (i00%qk)/qr; // quant index+ const int iybs = i00 - i00%qk; // dst block start index+ const int y_offset = qr == 1 ? 1 : qk/2;- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);- const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1));+ // dequantize+ dfloat2 v;+ dequantize_kernel(src0_row, ib, iqs, v);- int qs0 = (ql >> 0) & 0x0F0F0F0F;- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28+ dst_row[iybs + iqs + 0] = v.x;+ dst_row[iybs + iqs + y_offset] = v.y;+}- x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;+template<typename src0_t, typename dst_t>+static __global__ void k_get_rows_float(+ const src0_t * src0, const int32_t * src1, dst_t * dst,+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {- int qs1 = (ql >> 4) & 0x0F0F0F0F;- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28+ const int i00 = blockIdx.x*blockDim.x + threadIdx.x;+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;- x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;+ if (i00 >= ne00) {+ return;}- const int blocks_per_tile_x_row = WARP_SIZE / QI5_1;- const int kbxd = k % blocks_per_tile_x_row;+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {- int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row;+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);- if (need_check) {- i = min(i, i_max);- }+ dst_row[i00] = src0_row[i00];+}- const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbxd;+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int k) {+ const int i = 2*(blockDim.x*blockIdx.x + threadIdx.x);- x_dm[i * (WARP_SIZE/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;+ if (i >= k) {+ return;}-}-static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh; (void)x_sc;+ const int ib = i/qk; // block index+ const int iqs = (i%qk)/qr; // quant index+ const int iybs = i - i%qk; // y block start index+ const int y_offset = qr == 1 ? 1 : qk/2;- const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));- const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1;+ // dequantize+ dfloat2 v;+ dequantize_kernel(vx, ib, iqs, v);- int u[2*VDR_Q5_1_Q8_1_MMQ];+ y[iybs + iqs + 0] = v.x;+ y[iybs + iqs + y_offset] = v.y;+}-#pragma unroll- for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {- u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];- u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_1) % WARP_SIZE];+template <typename src_t, typename dst_t>+static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __restrict__ y, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;}- return vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>- (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);+ const src_t * x = (src_t *) vx;++ y[i] = x[i];}-static __device__ __forceinline__ float vec_dot_q8_0_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+template <bool need_check>+static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int k) {+#if __CUDA_ARCH__ >= CC_PASCAL+ constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;- const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;+ const int i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;+ const int * x0 = ((int *) vx) + blockIdx.x * nint;+ half2 * y2 = (half2 *) (y + i0);- int v[VDR_Q8_0_Q8_1_MMVQ];- int u[VDR_Q8_0_Q8_1_MMVQ];+ __shared__ int vals[nint];#pragma unroll- for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_int8(bq8_0->qs, iqs + i);- u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {+ if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {+ break;+ }++ const int ix = ix0 + threadIdx.x;+ vals[ix] = x0[ix];}- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));-}+#pragma unroll+ for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {+ if (need_check && i0 + iy + 2*threadIdx.x >= k) {+ return;+ }-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh; (void)x_sc;+ const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);+ const half d = *b0;+ const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];- __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0];+ y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));+ }+#else+ GGML_UNUSED(vx);+ GGML_UNUSED(y);+ GGML_UNUSED(k);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_PASCAL+}- *x_ql = tile_x_qs;- *x_dm = (half2 *) tile_x_d;-}--template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh; (void)x_sc;+// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called+// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q- GGML_CUDA_ASSUME(i_offset >= 0);- GGML_CUDA_ASSUME(i_offset < nwarps);- GGML_CUDA_ASSUME(k >= 0);- GGML_CUDA_ASSUME(k < WARP_SIZE);+#define VDR_Q4_0_Q8_1_MMVQ 2+#define VDR_Q4_0_Q8_1_MMQ 4- const int kbx = k / QI8_0;- const int kqsx = k % QI8_0;- float * x_dmf = (float *) x_dm;+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(+ const int * v, const int * u, const float & d4, const half2 & ds8) {- const block_q8_0 * bx0 = (const block_q8_0 *) vx;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;+ for (int i = 0; i < vdr; ++i) {+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;- if (need_check) {- i = min(i, i_max);- }+ // SIMD dot product of quantized values+ sumi = __dp4a(vi0, u[2*i+0], sumi);+ sumi = __dp4a(vi1, u[2*i+1], sumi);+ }- const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbx;+ const float2 ds8f = __half22float2(ds8);- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_int8(bxi->qs, kqsx);- }+ // second part effectively subtracts 8 from each quant value+ return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const int blocks_per_tile_x_row = WARP_SIZE / QI8_0;- const int kbxd = k % blocks_per_tile_x_row;+#define VDR_Q4_1_Q8_1_MMVQ 2+#define VDR_Q4_1_Q8_1_MMQ 4-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) {- int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row;+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(+ const int * v, const int * u, const half2 & dm4, const half2 & ds8) {- if (need_check) {- i = min(i, i_max);- }+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;- const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbxd;+#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;- x_dmf[i * (WARP_SIZE/QI8_0) + i / QI8_0 + kbxd] = bxi->d;+ // SIMD dot product of quantized values+ sumi = __dp4a(vi0, u[2*i+0], sumi);+ sumi = __dp4a(vi1, u[2*i+1], sumi);}-}--static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh; (void)x_sc;- const float * x_dmf = (const float *) x_dm;- const float * y_df = (const float *) y_ds;+#ifdef GGML_CUDA_F16+ const float2 tmp = __half22float2(__hmul2(dm4, ds8));+ const float d4d8 = tmp.x;+ const float m4s8 = tmp.y;+#else+ const float2 dm4f = __half22float2(dm4);+ const float2 ds8f = __half22float2(ds8);+ const float d4d8 = dm4f.x * ds8f.x;+ const float m4s8 = dm4f.y * ds8f.y;+#endif // GGML_CUDA_F16- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>- (&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[j * WARP_SIZE + k], x_dmf[i * (WARP_SIZE/QI8_0) + i/QI8_0 + k/QI8_0],- y_df[j * (WARP_SIZE/QI8_1) + k/QI8_1]);+ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-static __device__ __forceinline__ float vec_dot_q2_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q2_K * bq2_K = (const block_q2_K *) vbq;-- const int bq8_offset = QR2_K * (iqs / QI8_1);- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);+#define VDR_Q5_0_Q8_1_MMVQ 2+#define VDR_Q5_0_Q8_1_MMQ 4- const uint8_t * scales = bq2_K->scales + scale_offset;+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(+ const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {- const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);- int u[QR2_K];- float d8[QR2_K];+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;#pragma unroll- for (int i = 0; i < QR2_K; ++ i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);- d8[i] = __low2half(bq8_1[bq8_offset + i].ds);- }-- return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);-}+ for (int i = 0; i < vdr; ++i) {+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28+ sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh;+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28+ sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values+ }- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];+ const float2 ds8f = __half22float2(ds8);- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_sc = tile_x_sc;+ // second part effectively subtracts 16 from each quant value+ return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh;-- GGML_CUDA_ASSUME(i_offset >= 0);- GGML_CUDA_ASSUME(i_offset < nwarps);- GGML_CUDA_ASSUME(k >= 0);- GGML_CUDA_ASSUME(k < WARP_SIZE);+#define VDR_Q5_1_Q8_1_MMVQ 2+#define VDR_Q5_1_Q8_1_MMQ 4- const int kbx = k / QI2_K;- const int kqsx = k % QI2_K;+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(+ const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {- const block_q2_K * bx0 = (const block_q2_K *) vx;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }-- const block_q2_K * bxi = bx0 + i*blocks_per_row + kbx;+ for (int i = 0; i < vdr; ++i) {+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28+ sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28+ sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values}- const int blocks_per_tile_x_row = WARP_SIZE / QI2_K;- const int kbxd = k % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) {- int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }+#ifdef GGML_CUDA_F16+ const float2 tmp = __half22float2(__hmul2(dm5, ds8));+ const float d5d8 = tmp.x;+ const float m5s8 = tmp.y;+#else+ const float2 dm5f = __half22float2(dm5);+ const float2 ds8f = __half22float2(ds8);+ const float d5d8 = dm5f.x * ds8f.x;+ const float m5s8 = dm5f.y * ds8f.y;+#endif // GGML_CUDA_F16- const block_q2_K * bxi = bx0 + i*blocks_per_row + kbxd;+ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);- x_dm[i * (WARP_SIZE/QI2_K) + i / QI2_K + kbxd] = bxi->dm;- }+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {- int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);+#define VDR_Q8_0_Q8_1_MMVQ 2+#define VDR_Q8_0_Q8_1_MMQ 8- if (need_check) {- i = min(i, i_max);- }+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(+ const int * v, const int * u, const float & d8_0, const float & d8_1) {- const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI2_K/4);+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;- x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));+#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ // SIMD dot product of quantized values+ sumi = __dp4a(v[i], u[i], sumi);}-}--static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh;- const int kbx = k / QI2_K;- const int ky = (k % QI2_K) * QR2_K;- const float * y_df = (const float *) y_ds;+ return d8_0*d8_1 * sumi;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- int v[QR2_K*VDR_Q2_K_Q8_1_MMQ];+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(+ const int * v, const int * u, const half2 & dm8, const half2 & ds8) {- const int kqsx = i * (WARP_SIZE + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);- const int shift = 2 * ((ky % (2*QI2_K)) / (QI2_K/2));+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;#pragma unroll- for (int l = 0; l < QR2_K*VDR_Q2_K_Q8_1_MMQ; ++l) {- v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;+ for (int i = 0; i < vdr; ++i) {+ // SIMD dot product of quantized values+ sumi = __dp4a(v[i], u[i], sumi);}- const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE/4) + i/4 + kbx*4]) + ky/4;-- const int index_y = j * WARP_SIZE + (QR2_K*k) % WARP_SIZE;- return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);-}+#ifdef GGML_CUDA_F16+ const float2 tmp = __half22float2(__hmul2(dm8, ds8));+ const float d8d8 = tmp.x;+ const float m8s8 = tmp.y;+#else+ const float2 dm8f = __half22float2(dm8);+ const float2 ds8f = __half22float2(ds8);+ const float d8d8 = dm8f.x * ds8f.x;+ const float m8s8 = dm8f.y * ds8f.y;+#endif // GGML_CUDA_F16-static __device__ __forceinline__ float vec_dot_q3_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const block_q3_K * bq3_K = (const block_q3_K *) vbq;+#define VDR_Q2_K_Q8_1_MMVQ 1+#define VDR_Q2_K_Q8_1_MMQ 2- const int bq8_offset = QR3_K * (iqs / (QI3_K/2));- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(+ const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const half2 & dm2, const float * __restrict__ d8) {- const float d = bq3_K->d;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;- const int vl = get_int_from_uint8(bq3_K->qs, iqs);+#pragma unroll+ for (int i = 0; i < QR2_K; ++i) {+ const int sc = scales[2*i];- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted- const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;+ const int vi = (v >> (2*i)) & 0x03030303;- int u[QR3_K];- float d8[QR3_K];+ sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product-#pragma unroll- for (int i = 0; i < QR3_K; ++i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);- d8[i] = __low2half(bq8_1[bq8_offset + i].ds);+ // fill int with 4x m+ int m = sc >> 4;+ m |= m << 8;+ m |= m << 16;+ sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values}- return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);-}--template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q3_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {-- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI3_K) + mmq_y/QI3_K];- __shared__ int tile_x_qh[mmq_y * (WARP_SIZE/2) + mmq_y/2];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];+ const float2 dm2f = __half22float2(dm2);- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_qh = tile_x_qh;- *x_sc = tile_x_sc;+ return dm2f.x*sumf_d - dm2f.y*sumf_m;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q3_K(- const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,- int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const half2 & dm2, const float & d8) {- GGML_CUDA_ASSUME(i_offset >= 0);- GGML_CUDA_ASSUME(i_offset < nwarps);- GGML_CUDA_ASSUME(k >= 0);- GGML_CUDA_ASSUME(k < WARP_SIZE);+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi_d = 0;+ int sumi_m = 0;- const int kbx = k / QI3_K;- const int kqsx = k % QI3_K;+#pragma unroll+ for (int i0 = 0; i0 < QI8_1; i0 += QI8_1/2) {+ int sumi_d_sc = 0;- const block_q3_K * bx0 = (const block_q3_K *) vx;+ const int sc = scales[i0 / (QI8_1/2)];-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;+ // fill int with 4x m+ int m = sc >> 4;+ m |= m << 8;+ m |= m << 16;- if (need_check) {- i = min(i, i_max);+#pragma unroll+ for (int i = i0; i < i0 + QI8_1/2; ++i) {+ sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product+ sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m}- const block_q3_K * bxi = bx0 + i*blocks_per_row + kbx;-- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);+ sumi_d += sumi_d_sc * (sc & 0xF);}- const int blocks_per_tile_x_row = WARP_SIZE / QI3_K;- const int kbxd = k % blocks_per_tile_x_row;- float * x_dmf = (float *) x_dm;+ const float2 dm2f = __half22float2(dm2);-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) {- int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y;+ return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- if (need_check) {- i = min(i, i_max);- }+#define VDR_Q3_K_Q8_1_MMVQ 1+#define VDR_Q3_K_Q8_1_MMQ 2- const block_q3_K * bxi = bx0 + i*blocks_per_row + kbxd;+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(+ const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const int & scale_offset, const float & d3, const float * __restrict__ d8) {- x_dmf[i * (WARP_SIZE/QI3_K) + i / QI3_K + kbxd] = bxi->d;- }+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf = 0.0f;#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {- int i = i0 + i_offset * 2 + k / (WARP_SIZE/2);+ for (int i = 0; i < QR3_K; ++i) {+ const int isc = scale_offset + 2*i;- if (need_check) {- i = min(i, i_max);- }+ const int isc_low = isc % (QK_K/32);+ const int sc_shift_low = 4 * (isc / (QK_K/32));+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;- const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/2)) / (QI3_K/2);+ const int isc_high = isc % (QK_K/64);+ const int sc_shift_high = 2 * (isc / (QK_K/64));+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted- x_qh[i * (WARP_SIZE/2) + i / 2 + k % (WARP_SIZE/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));- }+ const int sc = (sc_low | sc_high) - 32;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {- int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);+ const int vil = (vl >> (2*i)) & 0x03030303;- if (need_check) {- i = min(i, i_max);- }+ const int vih = ((vh >> i) << 2) & 0x04040404;- const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI3_K/4);+ const int vi = __vsubss4(vil, vih);- const int ksc = k % (QI3_K/4);+ sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product+ }- const int ksc_low = ksc % (QI3_K/8);- const int shift_low = 4 * (ksc / (QI3_K/8));- const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;+ return d3 * sumf;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const int ksc_high = QI3_K/8;- const int shift_high = 2 * ksc;- const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,+ const float & d3, const float & d8) {- const int sc = __vsubss4(sc_low | sc_high, 0x20202020);+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;- x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = sc;+#pragma unroll+ for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {+ int sumi_sc = 0;++ for (int i = i0; i < i0 + QI8_1/2; ++i) {+ sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product+ }++ sumi += sumi_sc * scales[i0 / (QI8_1/2)];}-}-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ return d3*d8 * sumi;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const int kbx = k / QI3_K;- const int ky = (k % QI3_K) * QR3_K;- const float * x_dmf = (const float *) x_dm;- const float * y_df = (const float *) y_ds;+#define VDR_Q4_K_Q8_1_MMVQ 2+#define VDR_Q4_K_Q8_1_MMQ 8- const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {- int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;#pragma unroll- for (int l = 0; l < QR3_K*VDR_Q3_K_Q8_1_MMQ; ++l) {- const int kqsx = i * (WARP_SIZE + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);- const int shift = 2 * ((ky % 32) / 8);- const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;+ for (int i = 0; i < QR4_K; ++i) {+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;- const int vh = x_qh[i * (WARP_SIZE/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);- const int vlh = (vh << 2) & 0x04040404;+ const int dot1 = __dp4a(v1i, u[2*i+1], __dp4a(v0i, u[2*i+0], 0)); // SIMD dot product+ const int dot2 = __dp4a(0x01010101, u[2*i+1], __dp4a(0x01010101, u[2*i+0], 0)); // sum of u- v[l] = __vsubss4(vll, vlh);+ sumf_d += d8[i] * (dot1 * sc[i]);+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values}- const int index_y = j * WARP_SIZE + (k*QR3_K) % WARP_SIZE;- return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);-}+ const float2 dm4f = __half22float2(dm4);-static __device__ __forceinline__ float vec_dot_q4_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ return dm4f.x*sumf_d - dm4f.y*sumf_m;-#ifndef GGML_QKK_64- const block_q4_K * bq4_K = (const block_q4_K *) vbq;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- int v[2];- int u[2*QR4_K];- float d8[QR4_K];+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {- // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6- const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;- // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12- // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44- // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76- // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108+#pragma unroll+ for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {+ int sumi_d = 0;- const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));- v[0] = q4[0];- v[1] = q4[4];+#pragma unroll+ for (int j = 0; j < QI8_1; ++j) {+ sumi_d = __dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product+ }- const uint16_t * scales = (const uint16_t *)bq4_K->scales;- uint16_t aux[2];- const int j = bq8_offset/2;- if (j < 2) {- aux[0] = scales[j+0] & 0x3f3f;- aux[1] = scales[j+2] & 0x3f3f;- } else {- aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);- aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);+ const float2 ds8f = __half22float2(ds8[i]);++ sumf_d += ds8f.x * (sc[i] * sumi_d);+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val}- const uint8_t * sc = (const uint8_t *)aux;- const uint8_t * m = sc + 2;- for (int i = 0; i < QR4_K; ++i) {- const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2half(bq8i->ds);+ const float2 dm4f = __half22float2(dm4);++ return dm4f.x*sumf_d - dm4f.y*sumf_m;++#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}++#define VDR_Q5_K_Q8_1_MMVQ 2+#define VDR_Q5_K_Q8_1_MMQ 8++// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(+ const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {++#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;++#pragma unroll+ for (int i = 0; i < QR5_K; ++i) {+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;++ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;++ const int v0i = vl0i | vh0i;+ const int v1i = vl1i | vh1i;++ const int dot1 = __dp4a(v0i, u[2*i+0], __dp4a(v1i, u[2*i+1], 0)); // SIMD dot product+ const int dot2 = __dp4a(0x01010101, u[2*i+0], __dp4a(0x01010101, u[2*i+1], 0)); // sum of u++ sumf_d += d8[i] * (dot1 * sc[i]);+ sumf_m += d8[i] * (dot2 * m[i]);- const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);- u[2*i+0] = q8[0];- u[2*i+1] = q8[4];}- return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);+ const float2 dm5f = __half22float2(dm5);++ return dm5f.x*sumf_d - dm5f.y*sumf_m;#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const block_q4_K * bq4_K = (const block_q4_K *) vbq;+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsicsfloat sumf_d = 0.0f;float sumf_m = 0.0f;- uint16_t aux16[2];- const uint8_t * s = (const uint8_t *)aux16;+#pragma unroll+ for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {+ int sumi_d = 0;- const uint16_t * a = (const uint16_t *)bq4_K->scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;+#pragma unroll+ for (int j = 0; j < QI8_1; ++j) {+ sumi_d = __dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product+ }- const float dall = bq4_K->dm[0];- const float dmin = bq4_K->dm[1];+ const float2 ds8f = __half22float2(ds8[i]);- const float d8_1 = __low2float(bq8_1[0].ds);- const float d8_2 = __low2float(bq8_1[1].ds);+ sumf_d += ds8f.x * (sc[i] * sumi_d);+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val+ }- const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));- const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);- const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));- const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);+ const float2 dm4f = __half22float2(dm4);- const int * q4 = (const int *)bq4_K->qs + (iqs/2);- const int v1 = q4[0];- const int v2 = q4[4];+ return dm4f.x*sumf_d - dm4f.y*sumf_m;- const int dot1 = __dp4a(ui2, v2 & 0x0f0f0f0f, __dp4a(ui1, v1 & 0x0f0f0f0f, 0));- const int dot2 = __dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, __dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0));- const int dot3 = __dp4a(0x01010101, ui2, __dp4a(0x01010101, ui1, 0));- const int dot4 = __dp4a(0x01010101, ui4, __dp4a(0x01010101, ui3, 0));+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]);- sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]);+#define VDR_Q6_K_Q8_1_MMVQ 1+#define VDR_Q6_K_Q8_1_MMQ 8- return dall * sumf_d - dmin * sumf_m;+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(+ const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,+ const float & d, const float * __restrict__ d8) {++#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf = 0.0f;++#pragma unroll+ for (int i = 0; i < QR6_K; ++i) {+ const int sc = scales[4*i];++ const int vil = (vl >> (4*i)) & 0x0F0F0F0F;++ const int vih = ((vh >> (4*i)) << 4) & 0x30303030;++ const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32++ sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product+ }+ return d*sumf;#elseNO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}-#endif+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,+ const float & d6, const float * __restrict__ d8) {++#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;++#pragma unroll+ for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {+ int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale++#pragma unroll+ for (int i = i0; i < i0 + 2; ++i) {+ sumi_d.x = __dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product+ sumi_d.x = __dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product++ sumi_d.y = __dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product+ sumi_d.y = __dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product+ }++ sumf_d += d8[i0/4] * (sc[i0/2+0]*sumi_d.x + sc[i0/2+1]*sumi_d.y);+ }++ return d6 * sumf_d;++#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh;+static __device__ __forceinline__ float vec_dot_q4_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];+ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_sc = tile_x_sc;+ int v[VDR_Q4_0_Q8_1_MMVQ];+ int u[2*VDR_Q4_0_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {+ v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);+ }++ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);}-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh);+ GGML_UNUSED(x_sc);++ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI4_0) + mmq_y/QI4_0];++ *x_ql = tile_x_qs;+ *x_dm = (half2 *) tile_x_d;+}++template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh;-+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);GGML_CUDA_ASSUME(k >= 0);GGML_CUDA_ASSUME(k < WARP_SIZE);- const int kbx = k / QI4_K; // == 0 if QK_K == 256- const int kqsx = k % QI4_K; // == k if QK_K == 256+ const int kbx = k / QI4_0;+ const int kqsx = k % QI4_0;- const block_q4_K * bx0 = (const block_q4_K *) vx;+ const block_q4_0 * bx0 = (const block_q4_0 *) vx;++ float * x_dmf = (float *) x_dm;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -4794,180 +4173,92 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlini = min(i, i_max);}- const block_q4_K * bxi = bx0 + i*blocks_per_row + kbx;+ const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbx;- x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);+ // x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbx] = bxi->d;}- const int blocks_per_tile_x_row = WARP_SIZE / QI4_K; // == 1 if QK_K == 256- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_0;+ const int kbxd = k % blocks_per_tile_x_row;#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) {- int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y;+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {+ int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row;if (need_check) {i = min(i, i_max);}- const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd;+ const block_q4_0 * bxi = bx0 + i*blocks_per_row + kbxd;-#if QK_K == 256- x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm;-#else- x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = {bxi->dm[0], bxi->dm[1]};-#endif+ x_dmf[i * (WARP_SIZE/QI4_0) + i / QI4_0 + kbxd] = bxi->d;}+}-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {- int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }+static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI4_K/8);+ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));+ const float * x_dmf = (const float *) x_dm;- const int * scales = (const int *) bxi->scales;+ int u[2*VDR_Q4_0_Q8_1_MMQ];- const int ksc = k % (WARP_SIZE/8);-- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8- int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits- scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits-- x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;+#pragma unroll+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_0) % WARP_SIZE];}-}--static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(- const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,- const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh;- const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8);-- const int index_y = j * WARP_SIZE + (QR4_K*k) % WARP_SIZE;- return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[index_y], sc, sc+8,- x_dm[i * (WARP_SIZE/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);+ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>+ (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dmf[i * (WARP_SIZE/QI4_0) + i/QI4_0 + k/QI4_0],+ y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);}-static __device__ __forceinline__ float vec_dot_q5_K_q8_1(+static __device__ __forceinline__ float vec_dot_q4_1_q8_1(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#ifndef GGML_QKK_64- const block_q5_K * bq5_K = (const block_q5_K *) vbq;-- int vl[2];- int vh[2];- int u[2*QR5_K];- float d8[QR5_K];-- const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));- const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));- const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));-- vl[0] = ql[0];- vl[1] = ql[4];-- vh[0] = qh[0] >> bq8_offset;- vh[1] = qh[4] >> bq8_offset;+ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;- const uint16_t * scales = (const uint16_t *)bq5_K->scales;- uint16_t aux[2];- const int j = bq8_offset/2;- if (j < 2) {- aux[0] = scales[j+0] & 0x3f3f;- aux[1] = scales[j+2] & 0x3f3f;- } else {- aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);- aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);- }- const uint8_t * sc = (const uint8_t *)aux;- const uint8_t * m = sc + 2;+ int v[VDR_Q4_1_Q8_1_MMVQ];+ int u[2*VDR_Q4_1_Q8_1_MMVQ];#pragma unroll- for (int i = 0; i < QR5_K; ++i) {- const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2float(bq8i->ds);-- const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);- u[2*i+0] = q8[0];- u[2*i+1] = q8[4];+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {+ v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);}- return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);--#else--#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const block_q5_K * bq5_K = (const block_q5_K *) vbq;-- const int8_t * s = bq5_K->scales;-- const float d = bq5_K->d;-- const float d8_1 = __low2half(bq8_1[0].ds);- const float d8_2 = __low2half(bq8_1[1].ds);-- const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));- const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);- const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));- const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);-- const int * ql = (const int *)bq5_K->qs + (iqs/2);- const int vl1 = ql[0];- const int vl2 = ql[4];-- const int step = 4 * (iqs/2); // 0, 4, 8, 12- const int im = step/8; // = 0 for iqs = 0, 2, = 1 for iqs = 4, 6- const int in = step%8; // 0, 4, 0, 4- const int vh = (*((const int *)(bq5_K->qh + in))) >> im;-- const int v1 = (((vh << 4) & 0x10101010) ^ 0x10101010) | ((vl1 >> 0) & 0x0f0f0f0f);- const int v2 = (((vh << 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 0) & 0x0f0f0f0f);- const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f);- const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f);-- const float sumf_d = d8_1 * (__dp4a(ui1, v1, 0) * s[0] + __dp4a(ui2, v2, 0) * s[1])- + d8_2 * (__dp4a(ui3, v3, 0) * s[2] + __dp4a(ui4, v4, 0) * s[3]);-- return d * sumf_d;--#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A--#endif+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);}-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh;+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + + mmq_y];+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_1) + mmq_y/QI4_1];- *x_ql = tile_x_ql;+ *x_ql = tile_x_qs;*x_dm = tile_x_dm;- *x_sc = tile_x_sc;}-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh;+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);GGML_CUDA_ASSUME(k >= 0);GGML_CUDA_ASSUME(k < WARP_SIZE);- const int kbx = k / QI5_K; // == 0 if QK_K == 256- const int kqsx = k % QI5_K; // == k if QK_K == 256+ const int kbx = k / QI4_1;+ const int kqsx = k % QI4_1;- const block_q5_K * bx0 = (const block_q5_K *) vx;+ const block_q4_1 * bx0 = (const block_q4_1 *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -4977,129 +4268,92 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlini = min(i, i_max);}- const block_q5_K * bxi = bx0 + i*blocks_per_row + kbx;- const int ky = QR5_K*kqsx;-- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);- const int ql0 = (ql >> 0) & 0x0F0F0F0F;- const int ql1 = (ql >> 4) & 0x0F0F0F0F;-- const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K/4));- const int qh0 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 0)) << 4) & 0x10101010;- const int qh1 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 1)) << 4) & 0x10101010;-- const int kq0 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + 0;- const int kq1 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + (QI5_K/4);+ const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbx;- x_ql[i * (2*WARP_SIZE + 1) + kq0] = ql0 | qh0;- x_ql[i * (2*WARP_SIZE + 1) + kq1] = ql1 | qh1;+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);}- const int blocks_per_tile_x_row = WARP_SIZE / QI5_K; // == 1 if QK_K == 256- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) {- int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd;--#if QK_K == 256- x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm;-#endif- }+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_1;+ const int kbxd = k % blocks_per_tile_x_row;#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {- int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {+ int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row;if (need_check) {i = min(i, i_max);}- const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI5_K/8);-- const int * scales = (const int *) bxi->scales;-- const int ksc = k % (WARP_SIZE/8);-- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8- int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits- scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits+ const block_q4_1 * bxi = bx0 + i*blocks_per_row + kbxd;- x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;+ x_dm[i * (WARP_SIZE/QI4_1) + i / QI4_1 + kbxd] = bxi->dm;}}-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(+static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh;-- const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- const int index_x = i * (QR5_K*WARP_SIZE + 1) + QR5_K*k;- const int index_y = j * WARP_SIZE + (QR5_K*k) % WARP_SIZE;- return vec_dot_q5_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, sc+8,- x_dm[i * (WARP_SIZE/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);-}+ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));-static __device__ __forceinline__ float vec_dot_q6_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ int u[2*VDR_Q4_1_Q8_1_MMQ];- const block_q6_K * bq6_K = (const block_q6_K *) vbq;+#pragma unroll+ for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI4_1) % WARP_SIZE];+ }- const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);- const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);- const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>+ (&x_ql[i * (WARP_SIZE + 1) + k], u, x_dm[i * (WARP_SIZE/QI4_1) + i/QI4_1 + k/QI4_1],+ y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);+}- const int vl = get_int_from_uint8(bq6_K->ql, iqs);- const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;+static __device__ __forceinline__ float vec_dot_q5_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- const int8_t * scales = bq6_K->scales + scale_offset;+ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;- int u[QR6_K];- float d8[QR6_K];+ int vl[VDR_Q5_0_Q8_1_MMVQ];+ int vh[VDR_Q5_0_Q8_1_MMVQ];+ int u[2*VDR_Q5_0_Q8_1_MMVQ];#pragma unroll- for (int i = 0; i < QR6_K; ++i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);- d8[i] = __low2half(bq8_1[bq8_offset + 2*i].ds);+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {+ vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);+ vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);}- return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);+ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);}-template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {- (void)x_qh;+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI5_0) + mmq_y/QI5_0];*x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_sc = tile_x_sc;+ *x_dm = (half2 *) tile_x_d;}-template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {- (void)x_qh;+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);GGML_CUDA_ASSUME(i_offset >= 0);GGML_CUDA_ASSUME(i_offset < nwarps);GGML_CUDA_ASSUME(k >= 0);GGML_CUDA_ASSUME(k < WARP_SIZE);- const int kbx = k / QI6_K; // == 0 if QK_K == 256- const int kqsx = k % QI6_K; // == k if QK_K == 256+ const int kbx = k / QI5_0;+ const int kqsx = k % QI5_0;- const block_q6_K * bx0 = (const block_q6_K *) vx;+ const block_q5_0 * bx0 = (const block_q5_0 *) vx;#pragma unrollfor (int i0 = 0; i0 < mmq_y; i0 += nwarps) {@@ -5109,7097 +4363,7075 @@ template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinlini = min(i, i_max);}- const block_q6_K * bxi = bx0 + i*blocks_per_row + kbx;- const int ky = QR6_K*kqsx;+ const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbx;- const int ql = get_int_from_uint8(bxi->ql, kqsx);- const int ql0 = (ql >> 0) & 0x0F0F0F0F;- const int ql1 = (ql >> 4) & 0x0F0F0F0F;+ const int ql = get_int_from_uint8(bxi->qs, kqsx);+ const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0));- const int qh = get_int_from_uint8(bxi->qh, (QI6_K/4) * (kqsx / (QI6_K/2)) + kqsx % (QI6_K/4));- const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) << 4) & 0x30303030;- const int qh1 = (qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) & 0x30303030;+ int qs0 = (ql >> 0) & 0x0F0F0F0F;+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28+ qs0 = __vsubss4(qs0, 0x10101010); // subtract 16- const int kq0 = ky - ky % QI6_K + k % (QI6_K/2) + 0;- const int kq1 = ky - ky % QI6_K + k % (QI6_K/2) + (QI6_K/2);+ x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;- x_ql[i * (2*WARP_SIZE + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);- x_ql[i * (2*WARP_SIZE + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);+ int qs1 = (ql >> 4) & 0x0F0F0F0F;+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28+ qs1 = __vsubss4(qs1, 0x10101010); // subtract 16++ x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;}- const int blocks_per_tile_x_row = WARP_SIZE / QI6_K; // == 1 if QK_K == 256- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256+ const int blocks_per_tile_x_row = WARP_SIZE / QI5_0;+ const int kbxd = k % blocks_per_tile_x_row;float * x_dmf = (float *) x_dm;#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {- int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y;+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {+ int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row;if (need_check) {i = min(i, i_max);}- const block_q6_K * bxi = bx0 + i*blocks_per_row + kbxd;+ const block_q5_0 * bxi = bx0 + i*blocks_per_row + kbxd;- x_dmf[i * (WARP_SIZE/QI6_K) + i / QI6_K + kbxd] = bxi->d;+ x_dmf[i * (WARP_SIZE/QI5_0) + i / QI5_0 + kbxd] = bxi->d;}+}++static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);++ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));+ const int index_bx = i * (WARP_SIZE/QI5_0) + i/QI5_0 + k/QI5_0;+ const float * x_dmf = (const float *) x_dm;+ const float * y_df = (const float *) y_ds;++ int u[2*VDR_Q5_0_Q8_1_MMQ];#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {- int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;+ for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_0) % WARP_SIZE];+ }++ return vec_dot_q8_0_q8_1_impl<QR5_0*VDR_Q5_0_Q8_1_MMQ>+ (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dmf[index_bx], y_df[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);+}++static __device__ __forceinline__ float vec_dot_q5_1_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {++ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;++ int vl[VDR_Q5_1_Q8_1_MMVQ];+ int vh[VDR_Q5_1_Q8_1_MMVQ];+ int u[2*VDR_Q5_1_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {+ vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);+ vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);+ }++ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);+}++template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_1(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);++ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_1) + mmq_y/QI5_1];++ *x_ql = tile_x_ql;+ *x_dm = tile_x_dm;+}++template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);++ GGML_CUDA_ASSUME(i_offset >= 0);+ GGML_CUDA_ASSUME(i_offset < nwarps);+ GGML_CUDA_ASSUME(k >= 0);+ GGML_CUDA_ASSUME(k < WARP_SIZE);++ const int kbx = k / QI5_1;+ const int kqsx = k % QI5_1;++ const block_q5_1 * bx0 = (const block_q5_1 *) vx;++#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {+ int i = i0 + i_offset;if (need_check) {i = min(i, i_max);}- const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / 4;+ const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbx;- x_sc[i * (WARP_SIZE/8) + i / 8 + k % (WARP_SIZE/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));+ const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);+ const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1));++ int qs0 = (ql >> 0) & 0x0F0F0F0F;+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28++ x_ql[i * (2*WARP_SIZE + 1) + 2*k+0] = qs0;++ int qs1 = (ql >> 4) & 0x0F0F0F0F;+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28++ x_ql[i * (2*WARP_SIZE + 1) + 2*k+1] = qs1;+ }++ const int blocks_per_tile_x_row = WARP_SIZE / QI5_1;+ const int kbxd = k % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {+ int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row;++ if (need_check) {+ i = min(i, i_max);+ }++ const block_q5_1 * bxi = bx0 + i*blocks_per_row + kbxd;++ x_dm[i * (WARP_SIZE/QI5_1) + i / QI5_1 + kbxd] = bxi->dm;}}-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(+static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {- (void)x_qh;+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- const float * x_dmf = (const float *) x_dm;- const float * y_df = (const float *) y_ds;+ const int kyqs = k % (QI8_1/2) + QI8_1 * (k / (QI8_1/2));+ const int index_bx = i * (WARP_SIZE/QI5_1) + + i/QI5_1 + k/QI5_1;- const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/8]);+ int u[2*VDR_Q5_1_Q8_1_MMQ];- const int index_x = i * (QR6_K*WARP_SIZE + 1) + QR6_K*k;- const int index_y = j * WARP_SIZE + (QR6_K*k) % WARP_SIZE;- return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);+#pragma unroll+ for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {+ u[2*l+0] = y_qs[j * WARP_SIZE + (kyqs + l) % WARP_SIZE];+ u[2*l+1] = y_qs[j * WARP_SIZE + (kyqs + l + QI5_1) % WARP_SIZE];+ }++ return vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>+ (&x_ql[i * (2*WARP_SIZE + 1) + 2 * k], u, x_dm[index_bx], y_ds[j * (WARP_SIZE/QI8_1) + (2*k/QI8_1) % (WARP_SIZE/QI8_1)]);}-static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(+static __device__ __forceinline__ float vec_dot_q8_0_q8_1(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if QK_K == 256- const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;-#if QR2_XXS == 8- const int ib32 = iqs;- const uint16_t * q2 = bq2->qs + 4*ib32;- const uint8_t * aux8 = (const uint8_t *)q2;- const int8_t * q8 = bq8_1[ib32].qs;- uint32_t aux32 = q2[2] | (q2[3] << 16);- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);- const uint8_t signs = ksigns_iq2xs[aux32 & 127];- for (int j = 0; j < 8; ++j) {- sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- aux32 >>= 7;- }- const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.25f;- return d * sumi;-#else- // iqs is 0...15- const int ib32 = iqs/2;- const int il = iqs%2;- const uint16_t * q2 = bq2->qs + 4*ib32;- const uint8_t * aux8 = (const uint8_t *)q2;- const uint8_t * grid1 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+0]);- const uint8_t * grid2 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+1]);- const uint32_t aux32 = q2[2] | (q2[3] << 16);- const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * __low2float(bq8_1[ib32].ds) * 0.25f;- const uint8_t signs1 = ksigns_iq2xs[(aux32 >> 14*il) & 127];- const uint8_t signs2 = ksigns_iq2xs[(aux32 >> (14*il + 7)) & 127];- const int8_t * q8 = bq8_1[ib32].qs + 16*il;- int sumi1 = 0, sumi2 = 0;- for (int j = 0; j < 8; ++j) {- sumi1 += q8[j+0] * grid1[j] * (signs1 & kmask_iq2xs[j] ? -1 : 1);- sumi2 += q8[j+8] * grid2[j] * (signs2 & kmask_iq2xs[j] ? -1 : 1);- }- return d * (sumi1 + sumi2);-#endif-#else- assert(false);- return 0.f;-#endif-}+ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;-static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;+ int v[VDR_Q8_0_Q8_1_MMVQ];+ int u[VDR_Q8_0_Q8_1_MMVQ];- const int ib32 = iqs;- const uint16_t * q2 = bq2->qs + 4*ib32;- const int8_t * q8 = bq8_1[ib32].qs;- const uint8_t ls1 = bq2->scales[ib32] & 0xf;- const uint8_t ls2 = bq2->scales[ib32] >> 4;- int sumi1 = 0;- for (int l = 0; l < 2; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));- const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));- const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);- const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);- sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);- sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);- q8 += 8;- }- int sumi2 = 0;- for (int l = 2; l < 4; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));- const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));- const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);- const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);- sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);- sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);- q8 += 8;+#pragma unroll+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {+ v[i] = get_int_from_int8(bq8_0->qs, iqs + i);+ u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);}- const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);-#else- (void) ksigns64;- assert(false);- return 0.f;-#endif-#else- (void) ksigns64;- assert(false);- return 0.f;-#endif++ return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));}-// TODO-static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq2_s * bq2 = (const block_iq2_s *) vbq;+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q8_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- const int ib32 = iqs;- const int8_t * q8 = bq8_1[ib32].qs;- const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;- const uint8_t ls1 = bq2->scales[ib32] & 0xf;- const uint8_t ls2 = bq2->scales[ib32] >> 4;- int sumi1 = 0;- for (int l = 0; l < 2; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);- sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);- sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);- q8 += 8;- }- int sumi2 = 0;- for (int l = 2; l < 4; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);- sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);- sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);- q8 += 8;- }- const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);-#else- (void) ksigns64;- assert(false);- return 0.f;-#endif-#else- (void) ksigns64;- assert(false);- return 0.f;-#endif+ __shared__ int tile_x_qs[mmq_y * (WARP_SIZE) + mmq_y];+ __shared__ float tile_x_d[mmq_y * (WARP_SIZE/QI8_0) + mmq_y/QI8_0];++ *x_ql = tile_x_qs;+ *x_dm = (half2 *) tile_x_d;}-static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- const int ib32 = iqs;- const uint8_t * q3 = bq2->qs + 8*ib32;- const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;- const int8_t * q8 = bq8_1[ib32].qs;- uint32_t aux32 = gas[0] | (gas[1] << 16);- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];- const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];- const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));- const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);- const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);- sumi = __dp4a(grid_l, *((int *)q8+0), sumi);- sumi = __dp4a(grid_h, *((int *)q8+1), sumi);- q8 += 8;- aux32 >>= 7;- }- const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;- return d * sumi;-#else- assert(false);- return 0.f;-#endif-#else- assert(false);- return 0.f;-#endif-}+ GGML_CUDA_ASSUME(i_offset >= 0);+ GGML_CUDA_ASSUME(i_offset < nwarps);+ GGML_CUDA_ASSUME(k >= 0);+ GGML_CUDA_ASSUME(k < WARP_SIZE);-// TODO: don't use lookup table for signs-static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq3_s * bq2 = (const block_iq3_s *) vbq;+ const int kbx = k / QI8_0;+ const int kqsx = k % QI8_0;+ float * x_dmf = (float *) x_dm;- const int ib32 = iqs;- const uint8_t * qs = bq2->qs + 8*ib32;- const int8_t * q8 = bq8_1[ib32].qs;- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint32_t * grid1 = iq3xs_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));- const uint32_t * grid2 = iq3xs_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));- uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);- sumi = __dp4a(grid_l, *((int *)q8+0), sumi);- sumi = __dp4a(grid_h, *((int *)q8+1), sumi);- q8 += 8;- }- const float d = (float)bq2->d * (0.5f + ((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds) * 0.5f;- return d * sumi;-#else- assert(false);- return 0.f;-#endif-#else- assert(false);- return 0.f;-#endif-}+ const block_q8_0 * bx0 = (const block_q8_0 *) vx;+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {+ int i = i0 + i_offset;-static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if QK_K == 256- const block_iq1_s * bq1 = (const block_iq1_s *) vbq;+ if (need_check) {+ i = min(i, i_max);+ }- const int ib32 = iqs;- int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0;- const uint8_t h1 = bq1->scales[2*ib32+0];- const uint8_t h2 = bq1->scales[2*ib32+1];-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const int * q8 = (const int *)bq8_1[ib32].qs;- const int * grid1 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5)));- const int * grid2 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1)));- const int * grid3 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5)));- const int * grid4 = (const int *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1)));- for (int j = 0; j < 2; ++j) {- sumi1 = __dp4a(q8[j+0], grid1[j], sumi1);- sumi2 = __dp4a(q8[j+2], grid2[j], sumi2);- sumi3 = __dp4a(q8[j+4], grid3[j], sumi3);- sumi4 = __dp4a(q8[j+6], grid4[j], sumi4);+ const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbx;++ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_int8(bxi->qs, kqsx);}-#else- const int8_t * q8 = bq8_1[ib32].qs;- const int8_t * grid1 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+0] | ((h1 & 0x08) << 5)));- const int8_t * grid2 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+1] | ((h1 & 0x80) << 1)));- const int8_t * grid3 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+2] | ((h2 & 0x08) << 5)));- const int8_t * grid4 = (const int8_t *)(iq1s_grid + (bq1->qs[4*ib32+3] | ((h2 & 0x80) << 1)));- for (int j = 0; j < 8; ++j) {- sumi1 += q8[j+ 0] * grid1[j];- sumi2 += q8[j+ 8] * grid2[j];- sumi3 += q8[j+16] * grid3[j];- sumi4 += q8[j+24] * grid4[j];++ const int blocks_per_tile_x_row = WARP_SIZE / QI8_0;+ const int kbxd = k % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) {+ int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row;++ if (need_check) {+ i = min(i, i_max);+ }++ const block_q8_0 * bxi = bx0 + i*blocks_per_row + kbxd;++ x_dmf[i * (WARP_SIZE/QI8_0) + i / QI8_0 + kbxd] = bxi->d;}-#endif- const float d = (float)bq1->d * __low2float(bq8_1[ib32].ds);- return d * (sumi1 * (2*(h1 & 7) + 1) + sumi2 * (2*((h1 >> 4) & 7) + 1) +- sumi3 * (2*(h2 & 7) + 1) + sumi4 * (2*((h2 >> 4) & 7) + 1));-#else- assert(false);- return 0.f;-#endif}-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,- int & val1, int & val2) {+static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ GGML_UNUSED(x_qh); GGML_UNUSED(x_sc);- uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;- aux32 = q4 & 0x0f0f0f0f;- uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);- uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);- val1 = v1 | (v2 << 16);- aux32 = (q4 >> 4) & 0x0f0f0f0f;- v1 = values[q8[0]] | (values[q8[1]] << 8);- v2 = values[q8[2]] | (values[q8[3]] << 8);- val2 = v1 | (v2 << 16);+ const float * x_dmf = (const float *) x_dm;+ const float * y_df = (const float *) y_ds;++ return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>+ (&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[j * WARP_SIZE + k], x_dmf[i * (WARP_SIZE/QI8_0) + i/QI8_0 + k/QI8_0],+ y_df[j * (WARP_SIZE/QI8_1) + k/QI8_1]);}-#endif-static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(+static __device__ __forceinline__ float vec_dot_q2_K_q8_1(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- const block_iq4_nl * bq = (const block_iq4_nl *) vbq;--#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;- const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;+ const block_q2_K * bq2_K = (const block_q2_K *) vbq;- const uint8_t * values = (const uint8_t *)kvalues_iq4nl;+ const int bq8_offset = QR2_K * (iqs / QI8_1);+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);- int v1, v2;- int sumi1 = 0, sumi2 = 0;- for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {- const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);- get_int_from_table_16(aux, values, v1, v2);- sumi1 = __dp4a(v1, q8[l+0], sumi1);- sumi2 = __dp4a(v2, q8[l+4], sumi2);- }+ const uint8_t * scales = bq2_K->scales + scale_offset;-#else- const uint8_t * q4 = bq->qs + 4*iqs;- const int8_t * q8 = bq8_1->qs + 4*iqs;+ const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);+ int u[QR2_K];+ float d8[QR2_K];- int sumi1 = 0, sumi2 = 0;- for (int l = 0; l < 4*VDR_Q4_0_Q8_1_MMVQ; ++l) {- sumi1 += q8[l+ 0] * kvalues_iq4nl[q4[l] & 0xf];- sumi2 += q8[l+16] * kvalues_iq4nl[q4[l] >> 4];+#pragma unroll+ for (int i = 0; i < QR2_K; ++ i) {+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);}-#endif- const float d = (float)bq->d * __low2float(bq8_1->ds);- return d * (sumi1 + sumi2);++ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);}-static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q2_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh);-#if QK_K == 256-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI2_K) + mmq_y/QI2_K];+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];- const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;- const uint8_t * values = (const uint8_t *)kvalues_iq4nl;+ *x_ql = tile_x_ql;+ *x_dm = tile_x_dm;+ *x_sc = tile_x_sc;+}- //// iqs is 0...7- //const int ib64 = iqs/2;- //const int il = iqs%2;- //const int32_t * q8_1 = (const int *)bq8_1[2*ib64+0].qs + 2*il;- //const int32_t * q8_2 = (const int *)bq8_1[2*ib64+1].qs + 2*il;- //const uint32_t * q4_1 = (const uint32_t *)bq4->qs + 8*ib64 + 2*il;- //const uint32_t * q4_2 = q4_1 + 4;- //const int8_t ls1 = (bq4->scales_l[ib64] & 0xf) | (((bq4->scales_h >> (4*ib64+0)) & 3) << 4);- //const int8_t ls2 = (bq4->scales_l[ib64] >> 4) | (((bq4->scales_h >> (4*ib64+2)) & 3) << 4);- //const float d1 = (float)bq4->d * (ls1 - 32) * __low2float(bq8_1[2*ib64+0].ds);- //const float d2 = (float)bq4->d * (ls2 - 32) * __low2float(bq8_1[2*ib64+1].ds);- //int v1, v2;- //int sumi1 = 0, sumi2 = 0;- //for (int j = 0; j < 2; ++j) {- // get_int_from_table_16(q4_1[j], values, v1, v2);- // sumi1 = __dp4a(v2, q8_1[j+4], __dp4a(v1, q8_1[j+0], sumi1));- // get_int_from_table_16(q4_2[j], values, v1, v2);- // sumi2 = __dp4a(v2, q8_2[j+4], __dp4a(v1, q8_2[j+0], sumi2));- //}- //return d1 * sumi1 + d2 * sumi2;+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ GGML_UNUSED(x_qh);- // iqs is 0...7- const int ib32 = iqs;- const int32_t * q8 = (const int *)bq8_1[ib32].qs;- const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;- const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);- const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds);- int v1, v2;- int sumi1 = 0, sumi2 = 0;- for (int j = 0; j < 4; ++j) {- get_int_from_table_16(q4[j], values, v1, v2);- sumi1 = __dp4a(v1, q8[j+0], sumi1);- sumi2 = __dp4a(v2, q8[j+4], sumi2);+ GGML_CUDA_ASSUME(i_offset >= 0);+ GGML_CUDA_ASSUME(i_offset < nwarps);+ GGML_CUDA_ASSUME(k >= 0);+ GGML_CUDA_ASSUME(k < WARP_SIZE);++ const int kbx = k / QI2_K;+ const int kqsx = k % QI2_K;++ const block_q2_K * bx0 = (const block_q2_K *) vx;++#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {+ int i = i0 + i_offset;++ if (need_check) {+ i = min(i, i_max);+ }++ const block_q2_K * bxi = bx0 + i*blocks_per_row + kbx;++ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);}- return d * (sumi1 + sumi2);- //// iqs is 0...15- //const int ib32 = iqs/2;- //const int il = iqs%2;- //const int32_t * q8 = (const int *)bq8_1[ib32].qs + 2*il;- //const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32 + 2*il;- //const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);- //const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds);- //int v1, v2;- //int sumi1 = 0, sumi2 = 0;- //for (int j = 0; j < 2; ++j) {- // get_int_from_table_16(q4[j], values, v1, v2);- // sumi1 = __dp4a(v1, q8[j+0], sumi1);- // sumi2 = __dp4a(v2, q8[j+4], sumi2);- //}- //return d * (sumi1 + sumi2);-#else- assert(false);- return 0.f;-#endif-#else- assert(false);- return 0.f;-#endif-}+ const int blocks_per_tile_x_row = WARP_SIZE / QI2_K;+ const int kbxd = k % blocks_per_tile_x_row;-template <int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,- allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>-static __device__ __forceinline__ void mul_mat_q(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) {+ int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y;- const block_q_t * x = (const block_q_t *) vx;- const block_q8_1 * y = (const block_q8_1 *) vy;+ if (need_check) {+ i = min(i, i_max);+ }- const int blocks_per_row_x = ncols_x / qk;- const int blocks_per_col_y = nrows_y / QK8_1;- const int blocks_per_warp = WARP_SIZE / qi;+ const block_q2_K * bxi = bx0 + i*blocks_per_row + kbxd;- const int & ncols_dst = ncols_y;+ x_dm[i * (WARP_SIZE/QI2_K) + i / QI2_K + kbxd] = bxi->dm;+ }- const int row_dst_0 = blockIdx.x*mmq_y;- const int & row_x_0 = row_dst_0;+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {+ int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);- const int col_dst_0 = blockIdx.y*mmq_x;- const int & col_y_0 = col_dst_0;+ if (need_check) {+ i = min(i, i_max);+ }- int * tile_x_ql = nullptr;- half2 * tile_x_dm = nullptr;- int * tile_x_qh = nullptr;- int * tile_x_sc = nullptr;+ const block_q2_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI2_K/4);- allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);+ x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = get_int_from_uint8_aligned(bxi->scales, k % (QI2_K/4));+ }+}- __shared__ int tile_y_qs[mmq_x * WARP_SIZE];- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE/QI8_1];+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ GGML_UNUSED(x_qh);- float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {{0.0f}};+ const int kbx = k / QI2_K;+ const int ky = (k % QI2_K) * QR2_K;+ const float * y_df = (const float *) y_ds;- for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {+ int v[QR2_K*VDR_Q2_K_Q8_1_MMQ];- load_tiles(x + row_x_0*blocks_per_row_x + ib0, tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc,- threadIdx.y, nrows_x-row_x_0-1, threadIdx.x, blocks_per_row_x);+ const int kqsx = i * (WARP_SIZE + 1) + kbx*QI2_K + (QI2_K/2) * (ky/(2*QI2_K)) + ky % (QI2_K/2);+ const int shift = 2 * ((ky % (2*QI2_K)) / (QI2_K/2));#pragma unroll- for (int ir = 0; ir < qr; ++ir) {- const int kqs = ir*WARP_SIZE + threadIdx.x;- const int kbxd = kqs / QI8_1;+ for (int l = 0; l < QR2_K*VDR_Q2_K_Q8_1_MMQ; ++l) {+ v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;+ }-#pragma unroll- for (int i = 0; i < mmq_x; i += nwarps) {- const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses+ const uint8_t * scales = ((const uint8_t *) &x_sc[i * (WARP_SIZE/4) + i/4 + kbx*4]) + ky/4;- const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];+ const int index_y = j * WARP_SIZE + (QR2_K*k) % WARP_SIZE;+ return vec_dot_q2_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE/QI2_K) + i/QI2_K + kbx], y_df[index_y/QI8_1]);+}- const int index_y = (threadIdx.y + i) * WARP_SIZE + kqs % WARP_SIZE;- tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);- }+static __device__ __forceinline__ float vec_dot_q3_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#pragma unroll- for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {- const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE/QI8_1)) % mmq_x;- const int kby = threadIdx.x % (WARP_SIZE/QI8_1);- const int col_y_eff = min(col_y_0 + ids, ncols_y-1);+ const block_q3_K * bq3_K = (const block_q3_K *) vbq;- // if the sum is not needed it's faster to transform the scale to f32 ahead of time- const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE/QI8_1) + kby].ds;- half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE/QI8_1) + kby];- if (need_sum) {- *dsi_dst = *dsi_src;- } else {- float * dfi_dst = (float *) dsi_dst;- *dfi_dst = __low2half(*dsi_src);- }- }+ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);- __syncthreads();+ const float d = bq3_K->d;++ const int vl = get_int_from_uint8(bq3_K->qs, iqs);++ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted+ const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;++ int u[QR3_K];+ float d8[QR3_K];-// #pragma unroll // unrolling this loop causes too much register pressure- for (int k = ir*WARP_SIZE/qr; k < (ir+1)*WARP_SIZE/qr; k += vdr) {#pragma unroll- for (int j = 0; j < mmq_x; j += nwarps) {+ for (int i = 0; i < QR3_K; ++i) {+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);+ }++ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);+}++template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q3_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {++ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI3_K) + mmq_y/QI3_K];+ __shared__ int tile_x_qh[mmq_y * (WARP_SIZE/2) + mmq_y/2];+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/4) + mmq_y/4];++ *x_ql = tile_x_ql;+ *x_dm = tile_x_dm;+ *x_qh = tile_x_qh;+ *x_sc = tile_x_sc;+}++template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q3_K(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {++ GGML_CUDA_ASSUME(i_offset >= 0);+ GGML_CUDA_ASSUME(i_offset < nwarps);+ GGML_CUDA_ASSUME(k >= 0);+ GGML_CUDA_ASSUME(k < WARP_SIZE);++ const int kbx = k / QI3_K;+ const int kqsx = k % QI3_K;++ const block_q3_K * bx0 = (const block_q3_K *) vx;+#pragma unroll- for (int i = 0; i < mmq_y; i += WARP_SIZE) {- sum[i/WARP_SIZE][j/nwarps] += vec_dot(- tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,- threadIdx.x + i, threadIdx.y + j, k);- }- }- }+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {+ int i = i0 + i_offset;- __syncthreads();+ if (need_check) {+ i = min(i, i_max);}++ const block_q3_K * bxi = bx0 + i*blocks_per_row + kbx;++ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);}+ const int blocks_per_tile_x_row = WARP_SIZE / QI3_K;+ const int kbxd = k % blocks_per_tile_x_row;+ float * x_dmf = (float *) x_dm;+#pragma unroll- for (int j = 0; j < mmq_x; j += nwarps) {- const int col_dst = col_dst_0 + j + threadIdx.y;+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) {+ int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y;- if (col_dst >= ncols_dst) {- return;+ if (need_check) {+ i = min(i, i_max);}-#pragma unroll- for (int i = 0; i < mmq_y; i += WARP_SIZE) {- const int row_dst = row_dst_0 + threadIdx.x + i;+ const block_q3_K * bxi = bx0 + i*blocks_per_row + kbxd;- if (row_dst >= nrows_dst) {- continue;- }+ x_dmf[i * (WARP_SIZE/QI3_K) + i / QI3_K + kbxd] = bxi->d;+ }- dst[col_dst*nrows_dst + row_dst] = sum[i/WARP_SIZE][j/nwarps];+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {+ int i = i0 + i_offset * 2 + k / (WARP_SIZE/2);++ if (need_check) {+ i = min(i, i_max);}++ const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/2)) / (QI3_K/2);++ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted+ x_qh[i * (WARP_SIZE/2) + i / 2 + k % (WARP_SIZE/2)] = ~get_int_from_uint8(bxi->hmask, k % (QI3_K/2));}-}-#define MMQ_X_Q4_0_RDNA2 64-#define MMQ_Y_Q4_0_RDNA2 128-#define NWARPS_Q4_0_RDNA2 8-#define MMQ_X_Q4_0_RDNA1 64-#define MMQ_Y_Q4_0_RDNA1 64-#define NWARPS_Q4_0_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q4_0_AMPERE 4-#define MMQ_Y_Q4_0_AMPERE 32-#define NWARPS_Q4_0_AMPERE 4-#else-#define MMQ_X_Q4_0_AMPERE 64-#define MMQ_Y_Q4_0_AMPERE 128-#define NWARPS_Q4_0_AMPERE 4-#endif-#define MMQ_X_Q4_0_PASCAL 64-#define MMQ_Y_Q4_0_PASCAL 64-#define NWARPS_Q4_0_PASCAL 8+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {+ int i = i0 + i_offset * 4 + k / (WARP_SIZE/4);-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q4_0_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- mul_mat_q4_0(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+ if (need_check) {+ i = min(i, i_max);+ }-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q4_0_RDNA2;- const int mmq_y = MMQ_Y_Q4_0_RDNA2;- const int nwarps = NWARPS_Q4_0_RDNA2;-#else- const int mmq_x = MMQ_X_Q4_0_RDNA1;- const int mmq_y = MMQ_Y_Q4_0_RDNA1;- const int nwarps = NWARPS_Q4_0_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+ const block_q3_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/4)) / (QI3_K/4);- mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ const int ksc = k % (QI3_K/4);-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q4_0_AMPERE;- const int mmq_y = MMQ_Y_Q4_0_AMPERE;- const int nwarps = NWARPS_Q4_0_AMPERE;+ const int ksc_low = ksc % (QI3_K/8);+ const int shift_low = 4 * (ksc / (QI3_K/8));+ const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;- mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ const int ksc_high = QI3_K/8;+ const int shift_high = 2 * ksc;+ const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q4_0_PASCAL;- const int mmq_y = MMQ_Y_Q4_0_PASCAL;- const int nwarps = NWARPS_Q4_0_PASCAL;+ const int sc = __vsubss4(sc_low | sc_high, 0x20202020);- mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q4_0_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+ x_sc[i * (WARP_SIZE/4) + i / 4 + k % (WARP_SIZE/4)] = sc;+ }}-#define MMQ_X_Q4_1_RDNA2 64-#define MMQ_Y_Q4_1_RDNA2 128-#define NWARPS_Q4_1_RDNA2 8-#define MMQ_X_Q4_1_RDNA1 64-#define MMQ_Y_Q4_1_RDNA1 64-#define NWARPS_Q4_1_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q4_1_AMPERE 4-#define MMQ_Y_Q4_1_AMPERE 32-#define NWARPS_Q4_1_AMPERE 4-#else-#define MMQ_X_Q4_1_AMPERE 64-#define MMQ_Y_Q4_1_AMPERE 128-#define NWARPS_Q4_1_AMPERE 4-#endif-#define MMQ_X_Q4_1_PASCAL 64-#define MMQ_Y_Q4_1_PASCAL 64-#define NWARPS_Q4_1_PASCAL 8+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_PASCAL, 2)-#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q4_1(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+ const int kbx = k / QI3_K;+ const int ky = (k % QI3_K) * QR3_K;+ const float * x_dmf = (const float *) x_dm;+ const float * y_df = (const float *) y_ds;-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q4_1_RDNA2;- const int mmq_y = MMQ_Y_Q4_1_RDNA2;- const int nwarps = NWARPS_Q4_1_RDNA2;-#else- const int mmq_x = MMQ_X_Q4_1_RDNA1;- const int mmq_y = MMQ_Y_Q4_1_RDNA1;- const int nwarps = NWARPS_Q4_1_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+ const int8_t * scales = ((const int8_t *) (x_sc + i * (WARP_SIZE/4) + i/4 + kbx*4)) + ky/4;- mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ int v[QR3_K*VDR_Q3_K_Q8_1_MMQ];-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q4_1_AMPERE;- const int mmq_y = MMQ_Y_Q4_1_AMPERE;- const int nwarps = NWARPS_Q4_1_AMPERE;+#pragma unroll+ for (int l = 0; l < QR3_K*VDR_Q3_K_Q8_1_MMQ; ++l) {+ const int kqsx = i * (WARP_SIZE + 1) + kbx*QI3_K + (QI3_K/2) * (ky/(2*QI3_K)) + ky % (QI3_K/2);+ const int shift = 2 * ((ky % 32) / 8);+ const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;- mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ const int vh = x_qh[i * (WARP_SIZE/2) + i/2 + kbx * (QI3_K/2) + (ky+l)%8] >> ((ky+l) / 8);+ const int vlh = (vh << 2) & 0x04040404;-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q4_1_PASCAL;- const int mmq_y = MMQ_Y_Q4_1_PASCAL;- const int nwarps = NWARPS_Q4_1_PASCAL;+ v[l] = __vsubss4(vll, vlh);+ }- mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q4_1_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+ const int index_y = j * WARP_SIZE + (k*QR3_K) % WARP_SIZE;+ return vec_dot_q3_K_q8_1_impl_mmq(v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE/QI3_K) + i/QI3_K + kbx], y_df[index_y/QI8_1]);}-#define MMQ_X_Q5_0_RDNA2 64-#define MMQ_Y_Q5_0_RDNA2 128-#define NWARPS_Q5_0_RDNA2 8-#define MMQ_X_Q5_0_RDNA1 64-#define MMQ_Y_Q5_0_RDNA1 64-#define NWARPS_Q5_0_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q5_0_AMPERE 4-#define MMQ_Y_Q5_0_AMPERE 32-#define NWARPS_Q5_0_AMPERE 4-#else-#define MMQ_X_Q5_0_AMPERE 128-#define MMQ_Y_Q5_0_AMPERE 64-#define NWARPS_Q5_0_AMPERE 4-#endif-#define MMQ_X_Q5_0_PASCAL 64-#define MMQ_Y_Q5_0_PASCAL 64-#define NWARPS_Q5_0_PASCAL 8+static __device__ __forceinline__ float vec_dot_q4_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q5_0_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- mul_mat_q5_0(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+#ifndef GGML_QKK_64+ const block_q4_K * bq4_K = (const block_q4_K *) vbq;-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q5_0_RDNA2;- const int mmq_y = MMQ_Y_Q5_0_RDNA2;- const int nwarps = NWARPS_Q5_0_RDNA2;-#else- const int mmq_x = MMQ_X_Q5_0_RDNA1;- const int mmq_y = MMQ_Y_Q5_0_RDNA1;- const int nwarps = NWARPS_Q5_0_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+ int v[2];+ int u[2*QR4_K];+ float d8[QR4_K];- mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6+ const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q5_0_AMPERE;- const int mmq_y = MMQ_Y_Q5_0_AMPERE;- const int nwarps = NWARPS_Q5_0_AMPERE;+ // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12+ // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44+ // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76+ // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108- mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));+ v[0] = q4[0];+ v[1] = q4[4];-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q5_0_PASCAL;- const int mmq_y = MMQ_Y_Q5_0_PASCAL;- const int nwarps = NWARPS_Q5_0_PASCAL;+ const uint16_t * scales = (const uint16_t *)bq4_K->scales;+ uint16_t aux[2];+ const int j = bq8_offset/2;+ if (j < 2) {+ aux[0] = scales[j+0] & 0x3f3f;+ aux[1] = scales[j+2] & 0x3f3f;+ } else {+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);+ }+ const uint8_t * sc = (const uint8_t *)aux;+ const uint8_t * m = sc + 2;- mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q5_0_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}+ for (int i = 0; i < QR4_K; ++i) {+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);++ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);+ u[2*i+0] = q8[0];+ u[2*i+1] = q8[4];+ }++ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);-#define MMQ_X_Q5_1_RDNA2 64-#define MMQ_Y_Q5_1_RDNA2 128-#define NWARPS_Q5_1_RDNA2 8-#define MMQ_X_Q5_1_RDNA1 64-#define MMQ_Y_Q5_1_RDNA1 64-#define NWARPS_Q5_1_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q5_1_AMPERE 4-#define MMQ_Y_Q5_1_AMPERE 32-#define NWARPS_Q5_1_AMPERE 4#else-#define MMQ_X_Q5_1_AMPERE 128-#define MMQ_Y_Q5_1_AMPERE 64-#define NWARPS_Q5_1_AMPERE 4-#endif-#define MMQ_X_Q5_1_PASCAL 64-#define MMQ_Y_Q5_1_PASCAL 64-#define NWARPS_Q5_1_PASCAL 8-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q5_1_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-mul_mat_q5_1(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const block_q4_K * bq4_K = (const block_q4_K *) vbq;-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q5_1_RDNA2;- const int mmq_y = MMQ_Y_Q5_1_RDNA2;- const int nwarps = NWARPS_Q5_1_RDNA2;-#else- const int mmq_x = MMQ_X_Q5_1_RDNA1;- const int mmq_y = MMQ_Y_Q5_1_RDNA1;- const int nwarps = NWARPS_Q5_1_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;- mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ uint16_t aux16[2];+ const uint8_t * s = (const uint8_t *)aux16;-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q5_1_AMPERE;- const int mmq_y = MMQ_Y_Q5_1_AMPERE;- const int nwarps = NWARPS_Q5_1_AMPERE;+ const uint16_t * a = (const uint16_t *)bq4_K->scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;- mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ const float dall = bq4_K->dm[0];+ const float dmin = bq4_K->dm[1];-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q5_1_PASCAL;- const int mmq_y = MMQ_Y_Q5_1_PASCAL;- const int nwarps = NWARPS_Q5_1_PASCAL;+ const float d8_1 = __low2float(bq8_1[0].ds);+ const float d8_2 = __low2float(bq8_1[1].ds);++ const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));+ const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);+ const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));+ const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);++ const int * q4 = (const int *)bq4_K->qs + (iqs/2);+ const int v1 = q4[0];+ const int v2 = q4[4];++ const int dot1 = __dp4a(ui2, v2 & 0x0f0f0f0f, __dp4a(ui1, v1 & 0x0f0f0f0f, 0));+ const int dot2 = __dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, __dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0));+ const int dot3 = __dp4a(0x01010101, ui2, __dp4a(0x01010101, ui1, 0));+ const int dot4 = __dp4a(0x01010101, ui4, __dp4a(0x01010101, ui3, 0));++ sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]);+ sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]);++ return dall * sumf_d - dmin * sumf_m;- mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- (void) vec_dot_q5_1_q8_1_mul_mat;NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-#define MMQ_X_Q8_0_RDNA2 64-#define MMQ_Y_Q8_0_RDNA2 128-#define NWARPS_Q8_0_RDNA2 8-#define MMQ_X_Q8_0_RDNA1 64-#define MMQ_Y_Q8_0_RDNA1 64-#define NWARPS_Q8_0_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q8_0_AMPERE 4-#define MMQ_Y_Q8_0_AMPERE 32-#define NWARPS_Q8_0_AMPERE 4-#else-#define MMQ_X_Q8_0_AMPERE 128-#define MMQ_Y_Q8_0_AMPERE 64-#define NWARPS_Q8_0_AMPERE 4#endif-#define MMQ_X_Q8_0_PASCAL 64-#define MMQ_Y_Q8_0_PASCAL 64-#define NWARPS_Q8_0_PASCAL 8+}-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q8_0_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- mul_mat_q8_0(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh);-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q8_0_RDNA2;- const int mmq_y = MMQ_Y_Q8_0_RDNA2;- const int nwarps = NWARPS_Q8_0_RDNA2;-#else- const int mmq_x = MMQ_X_Q8_0_RDNA1;- const int mmq_y = MMQ_Y_Q8_0_RDNA1;- const int nwarps = NWARPS_Q8_0_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+ __shared__ int tile_x_ql[mmq_y * (WARP_SIZE) + mmq_y];+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI4_K) + mmq_y/QI4_K];+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];- mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ *x_ql = tile_x_ql;+ *x_dm = tile_x_dm;+ *x_sc = tile_x_sc;+}-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q8_0_AMPERE;- const int mmq_y = MMQ_Y_Q8_0_AMPERE;- const int nwarps = NWARPS_Q8_0_AMPERE;+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ GGML_UNUSED(x_qh);- mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ GGML_CUDA_ASSUME(i_offset >= 0);+ GGML_CUDA_ASSUME(i_offset < nwarps);+ GGML_CUDA_ASSUME(k >= 0);+ GGML_CUDA_ASSUME(k < WARP_SIZE);-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q8_0_PASCAL;- const int mmq_y = MMQ_Y_Q8_0_PASCAL;- const int nwarps = NWARPS_Q8_0_PASCAL;+ const int kbx = k / QI4_K; // == 0 if QK_K == 256+ const int kqsx = k % QI4_K; // == k if QK_K == 256- mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q8_0_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}+ const block_q4_K * bx0 = (const block_q4_K *) vx;-#define MMQ_X_Q2_K_RDNA2 64-#define MMQ_Y_Q2_K_RDNA2 128-#define NWARPS_Q2_K_RDNA2 8-#define MMQ_X_Q2_K_RDNA1 128-#define MMQ_Y_Q2_K_RDNA1 32-#define NWARPS_Q2_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q2_K_AMPERE 4-#define MMQ_Y_Q2_K_AMPERE 32-#define NWARPS_Q2_K_AMPERE 4-#else-#define MMQ_X_Q2_K_AMPERE 64-#define MMQ_Y_Q2_K_AMPERE 128-#define NWARPS_Q2_K_AMPERE 4-#endif-#define MMQ_X_Q2_K_PASCAL 64-#define MMQ_Y_Q2_K_PASCAL 64-#define NWARPS_Q2_K_PASCAL 8+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {+ int i = i0 + i_offset;-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q2_K_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-mul_mat_q2_K(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+ if (need_check) {+ i = min(i, i_max);+ }-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q2_K_RDNA2;- const int mmq_y = MMQ_Y_Q2_K_RDNA2;- const int nwarps = NWARPS_Q2_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q2_K_RDNA1;- const int mmq_y = MMQ_Y_Q2_K_RDNA1;- const int nwarps = NWARPS_Q2_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+ const block_q4_K * bxi = bx0 + i*blocks_per_row + kbx;- mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ x_ql[i * (WARP_SIZE + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);+ }-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q2_K_AMPERE;- const int mmq_y = MMQ_Y_Q2_K_AMPERE;- const int nwarps = NWARPS_Q2_K_AMPERE;+ const int blocks_per_tile_x_row = WARP_SIZE / QI4_K; // == 1 if QK_K == 256+ const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256- mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) {+ int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y;-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q2_K_PASCAL;- const int mmq_y = MMQ_Y_Q2_K_PASCAL;- const int nwarps = NWARPS_Q2_K_PASCAL;+ if (need_check) {+ i = min(i, i_max);+ }- mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q2_K_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}+ const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd;-#define MMQ_X_Q3_K_RDNA2 128-#define MMQ_Y_Q3_K_RDNA2 64-#define NWARPS_Q3_K_RDNA2 8-#define MMQ_X_Q3_K_RDNA1 32-#define MMQ_Y_Q3_K_RDNA1 128-#define NWARPS_Q3_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q3_K_AMPERE 4-#define MMQ_Y_Q3_K_AMPERE 32-#define NWARPS_Q3_K_AMPERE 4+#if QK_K == 256+ x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm;#else-#define MMQ_X_Q3_K_AMPERE 128-#define MMQ_Y_Q3_K_AMPERE 128-#define NWARPS_Q3_K_AMPERE 4+ x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = {bxi->dm[0], bxi->dm[1]};#endif-#define MMQ_X_Q3_K_PASCAL 64-#define MMQ_Y_Q3_K_PASCAL 64-#define NWARPS_Q3_K_PASCAL 8--template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_PASCAL, 2)-#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q3_K(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+ }-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q3_K_RDNA2;- const int mmq_y = MMQ_Y_Q3_K_RDNA2;- const int nwarps = NWARPS_Q3_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q3_K_RDNA1;- const int mmq_y = MMQ_Y_Q3_K_RDNA1;- const int nwarps = NWARPS_Q3_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;- mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ if (need_check) {+ i = min(i, i_max);+ }-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q3_K_AMPERE;- const int mmq_y = MMQ_Y_Q3_K_AMPERE;- const int nwarps = NWARPS_Q3_K_AMPERE;+ const block_q4_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI4_K/8);- mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ const int * scales = (const int *) bxi->scales;-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q3_K_PASCAL;- const int mmq_y = MMQ_Y_Q3_K_PASCAL;- const int nwarps = NWARPS_Q3_K_PASCAL;+ const int ksc = k % (WARP_SIZE/8);- mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q3_K_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+ // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8+ int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits+ scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits++ x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;+ }}-#define MMQ_X_Q4_K_RDNA2 64-#define MMQ_Y_Q4_K_RDNA2 128-#define NWARPS_Q4_K_RDNA2 8-#define MMQ_X_Q4_K_RDNA1 32-#define MMQ_Y_Q4_K_RDNA1 64-#define NWARPS_Q4_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q4_K_AMPERE 4-#define MMQ_Y_Q4_K_AMPERE 32-#define NWARPS_Q4_K_AMPERE 4-#else-#define MMQ_X_Q4_K_AMPERE 64-#define MMQ_Y_Q4_K_AMPERE 128-#define NWARPS_Q4_K_AMPERE 4-#endif-#define MMQ_X_Q4_K_PASCAL 64-#define MMQ_Y_Q4_K_PASCAL 64-#define NWARPS_Q4_K_PASCAL 8+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ GGML_UNUSED(x_qh);-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_PASCAL, 2)-#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q4_K(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2*((k % 16) / 8);-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q4_K_RDNA2;- const int mmq_y = MMQ_Y_Q4_K_RDNA2;- const int nwarps = NWARPS_Q4_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q4_K_RDNA1;- const int mmq_y = MMQ_Y_Q4_K_RDNA1;- const int nwarps = NWARPS_Q4_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+ const int index_y = j * WARP_SIZE + (QR4_K*k) % WARP_SIZE;+ return vec_dot_q4_K_q8_1_impl_mmq(&x_ql[i * (WARP_SIZE + 1) + k], &y_qs[index_y], sc, sc+8,+ x_dm[i * (WARP_SIZE/QI4_K) + i/QI4_K], &y_ds[index_y/QI8_1]);+}- mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+static __device__ __forceinline__ float vec_dot_q5_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q4_K_AMPERE;- const int mmq_y = MMQ_Y_Q4_K_AMPERE;- const int nwarps = NWARPS_Q4_K_AMPERE;+#ifndef GGML_QKK_64+ const block_q5_K * bq5_K = (const block_q5_K *) vbq;- mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ int vl[2];+ int vh[2];+ int u[2*QR5_K];+ float d8[QR5_K];-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q4_K_PASCAL;- const int mmq_y = MMQ_Y_Q4_K_PASCAL;- const int nwarps = NWARPS_Q4_K_PASCAL;+ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));- mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q4_K_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}+ vl[0] = ql[0];+ vl[1] = ql[4];-#define MMQ_X_Q5_K_RDNA2 64-#define MMQ_Y_Q5_K_RDNA2 128-#define NWARPS_Q5_K_RDNA2 8-#define MMQ_X_Q5_K_RDNA1 32-#define MMQ_Y_Q5_K_RDNA1 64-#define NWARPS_Q5_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q5_K_AMPERE 4-#define MMQ_Y_Q5_K_AMPERE 32-#define NWARPS_Q5_K_AMPERE 4-#else-#define MMQ_X_Q5_K_AMPERE 64-#define MMQ_Y_Q5_K_AMPERE 128-#define NWARPS_Q5_K_AMPERE 4-#endif-#define MMQ_X_Q5_K_PASCAL 64-#define MMQ_Y_Q5_K_PASCAL 64-#define NWARPS_Q5_K_PASCAL 8+ vh[0] = qh[0] >> bq8_offset;+ vh[1] = qh[4] >> bq8_offset;-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q5_K_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-mul_mat_q5_K(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;+ uint16_t aux[2];+ const int j = bq8_offset/2;+ if (j < 2) {+ aux[0] = scales[j+0] & 0x3f3f;+ aux[1] = scales[j+2] & 0x3f3f;+ } else {+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);+ }+ const uint8_t * sc = (const uint8_t *)aux;+ const uint8_t * m = sc + 2;++#pragma unroll+ for (int i = 0; i < QR5_K; ++i) {+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);++ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);+ u[2*i+0] = q8[0];+ u[2*i+1] = q8[4];+ }++ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q5_K_RDNA2;- const int mmq_y = MMQ_Y_Q5_K_RDNA2;- const int nwarps = NWARPS_Q5_K_RDNA2;#else- const int mmq_x = MMQ_X_Q5_K_RDNA1;- const int mmq_y = MMQ_Y_Q5_K_RDNA1;- const int nwarps = NWARPS_Q5_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)- mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const block_q5_K * bq5_K = (const block_q5_K *) vbq;-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q5_K_AMPERE;- const int mmq_y = MMQ_Y_Q5_K_AMPERE;- const int nwarps = NWARPS_Q5_K_AMPERE;+ const int8_t * s = bq5_K->scales;- mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ const float d = bq5_K->d;-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q5_K_PASCAL;- const int mmq_y = MMQ_Y_Q5_K_PASCAL;- const int nwarps = NWARPS_Q5_K_PASCAL;+ const float d8_1 = __low2half(bq8_1[0].ds);+ const float d8_2 = __low2half(bq8_1[1].ds);++ const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));+ const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);+ const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));+ const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);++ const int * ql = (const int *)bq5_K->qs + (iqs/2);+ const int vl1 = ql[0];+ const int vl2 = ql[4];++ const int step = 4 * (iqs/2); // 0, 4, 8, 12+ const int im = step/8; // = 0 for iqs = 0, 2, = 1 for iqs = 4, 6+ const int in = step%8; // 0, 4, 0, 4+ const int vh = (*((const int *)(bq5_K->qh + in))) >> im;++ const int v1 = (((vh << 4) & 0x10101010) ^ 0x10101010) | ((vl1 >> 0) & 0x0f0f0f0f);+ const int v2 = (((vh << 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 0) & 0x0f0f0f0f);+ const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f);+ const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f);++ const float sumf_d = d8_1 * (__dp4a(ui1, v1, 0) * s[0] + __dp4a(ui2, v2, 0) * s[1])+ + d8_2 * (__dp4a(ui3, v3, 0) * s[2] + __dp4a(ui4, v4, 0) * s[3]);++ return d * sumf_d;- mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- (void) vec_dot_q5_K_q8_1_mul_mat;NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-#define MMQ_X_Q6_K_RDNA2 64-#define MMQ_Y_Q6_K_RDNA2 128-#define NWARPS_Q6_K_RDNA2 8-#define MMQ_X_Q6_K_RDNA1 32-#define MMQ_Y_Q6_K_RDNA1 64-#define NWARPS_Q6_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q6_K_AMPERE 4-#define MMQ_Y_Q6_K_AMPERE 32-#define NWARPS_Q6_K_AMPERE 4-#else-#define MMQ_X_Q6_K_AMPERE 64-#define MMQ_Y_Q6_K_AMPERE 64-#define NWARPS_Q6_K_AMPERE 4#endif-#define MMQ_X_Q6_K_PASCAL 64-#define MMQ_Y_Q6_K_PASCAL 64-#define NWARPS_Q6_K_PASCAL 8+}-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_PASCAL, 2)-#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q6_K(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {--#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q6_K_RDNA2;- const int mmq_y = MMQ_Y_Q6_K_RDNA2;- const int nwarps = NWARPS_Q6_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q6_K_RDNA1;- const int mmq_y = MMQ_Y_Q6_K_RDNA1;- const int nwarps = NWARPS_Q6_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q5_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh);- mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI5_K) + mmq_y/QI5_K];+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q6_K_AMPERE;- const int mmq_y = MMQ_Y_Q6_K_AMPERE;- const int nwarps = NWARPS_Q6_K_AMPERE;+ *x_ql = tile_x_ql;+ *x_dm = tile_x_dm;+ *x_sc = tile_x_sc;+}- mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ GGML_UNUSED(x_qh);-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q6_K_PASCAL;- const int mmq_y = MMQ_Y_Q6_K_PASCAL;- const int nwarps = NWARPS_Q6_K_PASCAL;+ GGML_CUDA_ASSUME(i_offset >= 0);+ GGML_CUDA_ASSUME(i_offset < nwarps);+ GGML_CUDA_ASSUME(k >= 0);+ GGML_CUDA_ASSUME(k < WARP_SIZE);- mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- (void) vec_dot_q6_K_q8_1_mul_mat;- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}+ const int kbx = k / QI5_K; // == 0 if QK_K == 256+ const int kqsx = k % QI5_K; // == k if QK_K == 256-template <int ncols_y, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))-// tell the compiler to use as many registers as it wants, see nwarps definition below-__launch_bounds__((ncols_y <= 4 ? 4 : 2)*WARP_SIZE, 1)-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))-static __global__ void mul_mat_vec_q(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int nrows_y, const int nrows_dst) {+ const block_q5_K * bx0 = (const block_q5_K *) vx;-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3))- constexpr int nwarps = 1;- constexpr int rows_per_cuda_block = 1;-#else- constexpr int nwarps = ncols_y <= 4 ? 4 : 2;- constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2;-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3)+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {+ int i = i0 + i_offset;- const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;- const int row0 = rows_per_cuda_block*blockIdx.x;- const int blocks_per_row_x = ncols_x / qk;- const int blocks_per_col_y = nrows_y / QK8_1;- constexpr int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi;+ if (need_check) {+ i = min(i, i_max);+ }-// partial sum for each thread- float tmp[ncols_y][rows_per_cuda_block] = {0.0f};+ const block_q5_K * bxi = bx0 + i*blocks_per_row + kbx;+ const int ky = QR5_K*kqsx;- const block_q_t * x = (const block_q_t *) vx;- const block_q8_1 * y = (const block_q8_1 *) vy;+ const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;- for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {- const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx+ const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K/4));+ const int qh0 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 0)) << 4) & 0x10101010;+ const int qh1 = ((qh >> (2 * (kqsx / (QI5_K/4)) + 1)) << 4) & 0x10101010;- // x block quant index when casting the quants to int- const int kqs = vdr * (tid % (qi/vdr));+ const int kq0 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + 0;+ const int kq1 = ky - ky % (QI5_K/2) + k % (QI5_K/4) + (QI5_K/4);-#pragma unroll- for (int j = 0; j < ncols_y; ++j) {-#pragma unroll- for (int i = 0; i < rows_per_cuda_block; ++i) {- tmp[j][i] += vec_dot_q_cuda(- &x[kbx + (row0 + i)*blocks_per_row_x], &y[j*blocks_per_col_y + kby], kqs);- }- }+ x_ql[i * (2*WARP_SIZE + 1) + kq0] = ql0 | qh0;+ x_ql[i * (2*WARP_SIZE + 1) + kq1] = ql1 | qh1;}- __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y][rows_per_cuda_block][WARP_SIZE];- if (threadIdx.y > 0) {-#pragma unroll- for (int j = 0; j < ncols_y; ++j) {-#pragma unroll- for (int i = 0; i < rows_per_cuda_block; ++i) {- tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];- }- }- }- __syncthreads();- if (threadIdx.y > 0) {- return;- }+ const int blocks_per_tile_x_row = WARP_SIZE / QI5_K; // == 1 if QK_K == 256+ const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256- // sum up partial sums and write back result-#pragma unroll- for (int j = 0; j < ncols_y; ++j) {-#pragma unroll- for (int i = 0; i < rows_per_cuda_block; ++i) {#pragma unroll- for (int l = 0; l < nwarps-1; ++l) {- tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];- }- tmp[j][i] = warp_reduce_sum(tmp[j][i]);- }+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) {+ int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y;- if (threadIdx.x < rows_per_cuda_block) {- dst[j*nrows_dst + row0 + threadIdx.x] = tmp[j][threadIdx.x];+ if (need_check) {+ i = min(i, i_max);}- }-}-template <int qk, int qr, dequantize_kernel_t dequantize_kernel>-static __global__ void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows) {- // qk = quantized weights per x block- // qr = number of quantized weights per data value in x block- const int row = blockIdx.x*blockDim.y + threadIdx.y;+ const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd;- if (row >= nrows) {- return;+#if QK_K == 256+ x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm;+#endif}- const int tid = threadIdx.x;-- const int iter_stride = 2*GGML_CUDA_DMMV_X;- const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter- const int y_offset = qr == 1 ? 1 : qk/2;--// partial sum for each thread-#ifdef GGML_CUDA_F16- half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics-#else- float tmp = 0.0f;-#endif // GGML_CUDA_F16+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;- for (int i = 0; i < ncols; i += iter_stride) {- const int col = i + vals_per_iter*tid;- const int ib = (row*ncols + col)/qk; // x block index- const int iqs = (col%qk)/qr; // x quant index- const int iybs = col - col%qk; // y block start index+ if (need_check) {+ i = min(i, i_max);+ }-// processing >2 values per i iter is faster for fast GPUs-#pragma unroll- for (int j = 0; j < vals_per_iter; j += 2) {- // process 2 vals per j iter+ const block_q5_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / (QI5_K/8);- // dequantize- // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val- dfloat2 v;- dequantize_kernel(vx, ib, iqs + j/qr, v);+ const int * scales = (const int *) bxi->scales;- // matrix multiplication- // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2-#ifdef GGML_CUDA_F16- tmp += __hmul2(v, {- y[iybs + iqs + j/qr + 0],- y[iybs + iqs + j/qr + y_offset]- });-#else- tmp += v.x * y[iybs + iqs + j/qr + 0];- tmp += v.y * y[iybs + iqs + j/qr + y_offset];-#endif // GGML_CUDA_F16- }- }+ const int ksc = k % (WARP_SIZE/8);- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+ // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8+ int scales8 = (scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F; // lower 4 bits+ scales8 |= (scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits- if (tid == 0) {-#ifdef GGML_CUDA_F16- dst[row] = tmp.x + tmp.y;-#else- dst[row] = tmp;-#endif // GGML_CUDA_F16+ x_sc[i * (WARP_SIZE/8) + i / 8 + ksc] = scales8;}}-static __global__ void mul_mat_p021_f16_f32(- const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int nchannels_x, const int nchannels_y) {+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ GGML_UNUSED(x_qh);- const half * x = (const half *) vx;+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/16]) + 2 * ((k % 16) / 8);- const int row_x = blockDim.y*blockIdx.y + threadIdx.y;- const int channel = blockDim.z*blockIdx.z + threadIdx.z;- const int channel_x = channel / (nchannels_y / nchannels_x);+ const int index_x = i * (QR5_K*WARP_SIZE + 1) + QR5_K*k;+ const int index_y = j * WARP_SIZE + (QR5_K*k) % WARP_SIZE;+ return vec_dot_q5_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, sc+8,+ x_dm[i * (WARP_SIZE/QI5_K) + i/QI5_K], &y_ds[index_y/QI8_1]);+}- const int nrows_y = ncols_x;- const int nrows_dst = nrows_x;- const int row_dst = row_x;+static __device__ __forceinline__ float vec_dot_q6_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- float tmp = 0.0f;+ const block_q6_K * bq6_K = (const block_q6_K *) vbq;- for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {- const int col_x = col_x0 + threadIdx.x;+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));- if (col_x >= ncols_x) {- break;- }+ const int vl = get_int_from_uint8(bq6_K->ql, iqs);+ const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;- // x is transposed and permuted- const int ix = row_x*nchannels_x*ncols_x + channel_x*ncols_x + col_x;- const float xi = __half2float(x[ix]);+ const int8_t * scales = bq6_K->scales + scale_offset;- const int row_y = col_x;-- // y is not transposed but permuted- const int iy = channel*nrows_y + row_y;+ int u[QR6_K];+ float d8[QR6_K];- tmp += xi * y[iy];+#pragma unroll+ for (int i = 0; i < QR6_K; ++i) {+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);}- // dst is not transposed and not permuted- const int idst = channel*nrows_dst + row_dst;-- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);-- if (threadIdx.x == 0) {- dst[idst] = tmp;- }+ return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);}-static __global__ void mul_mat_vec_nc_f16_f32( // nc == non-contiguous- const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst, const int ncols_x, const int nrows_x,- const int row_stride_x, const int channel_stride_x, const int channel_x_divisor) {+template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q6_K(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {+ GGML_UNUSED(x_qh);- const half * x = (const half *) vx;+ __shared__ int tile_x_ql[mmq_y * (2*WARP_SIZE) + mmq_y];+ __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE/QI6_K) + mmq_y/QI6_K];+ __shared__ int tile_x_sc[mmq_y * (WARP_SIZE/8) + mmq_y/8];- const int row_x = blockDim.y*blockIdx.y + threadIdx.y;- const int channel = blockDim.z*blockIdx.z + threadIdx.z;- const int channel_x = channel / channel_x_divisor;+ *x_ql = tile_x_ql;+ *x_dm = tile_x_dm;+ *x_sc = tile_x_sc;+}- const int nrows_y = ncols_x;- const int nrows_dst = nrows_x;- const int row_dst = row_x;+template <int mmq_y, int nwarps, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(+ const void * __restrict__ vx, int * __restrict__ x_ql, half2 * __restrict__ x_dm, int * __restrict__ x_qh,+ int * __restrict__ x_sc, const int & i_offset, const int & i_max, const int & k, const int & blocks_per_row) {+ GGML_UNUSED(x_qh);- const int idst = channel*nrows_dst + row_dst;+ GGML_CUDA_ASSUME(i_offset >= 0);+ GGML_CUDA_ASSUME(i_offset < nwarps);+ GGML_CUDA_ASSUME(k >= 0);+ GGML_CUDA_ASSUME(k < WARP_SIZE);- float tmp = 0.0f;+ const int kbx = k / QI6_K; // == 0 if QK_K == 256+ const int kqsx = k % QI6_K; // == k if QK_K == 256- for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {- const int col_x = col_x0 + threadIdx.x;+ const block_q6_K * bx0 = (const block_q6_K *) vx;- if (col_x >= ncols_x) {- break;- }+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {+ int i = i0 + i_offset;- const int row_y = col_x;+ if (need_check) {+ i = min(i, i_max);+ }- const int ix = channel_x*channel_stride_x + row_x*row_stride_x + col_x;- const int iy = channel*nrows_y + row_y;+ const block_q6_K * bxi = bx0 + i*blocks_per_row + kbx;+ const int ky = QR6_K*kqsx;- const float xi = __half2float(x[ix]);+ const int ql = get_int_from_uint8(bxi->ql, kqsx);+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;- tmp += xi * y[iy];- }+ const int qh = get_int_from_uint8(bxi->qh, (QI6_K/4) * (kqsx / (QI6_K/2)) + kqsx % (QI6_K/4));+ const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) << 4) & 0x30303030;+ const int qh1 = (qh >> (2 * ((kqsx % (QI6_K/2)) / (QI6_K/4)))) & 0x30303030;- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+ const int kq0 = ky - ky % QI6_K + k % (QI6_K/2) + 0;+ const int kq1 = ky - ky % QI6_K + k % (QI6_K/2) + (QI6_K/2);- if (threadIdx.x == 0) {- dst[idst] = tmp;+ x_ql[i * (2*WARP_SIZE + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);+ x_ql[i * (2*WARP_SIZE + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);}-}--static __device__ void cpy_1_f32_f32(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- float * dsti = (float *) cdsti;-- *dsti = *xi;-}--static __device__ void cpy_1_f32_f16(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- half * dsti = (half *) cdsti;-- *dsti = __float2half(*xi);-}--static __device__ void cpy_1_f16_f16(const char * cxi, char * cdsti) {- const half * xi = (const half *) cxi;- half * dsti = (half *) cdsti;- *dsti = *xi;-}+ const int blocks_per_tile_x_row = WARP_SIZE / QI6_K; // == 1 if QK_K == 256+ const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256+ float * x_dmf = (float *) x_dm;-static __device__ void cpy_1_f16_f32(const char * cxi, char * cdsti) {- const half * xi = (const half *) cxi;- float * dsti = (float *) cdsti;+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {+ int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y;- *dsti = *xi;-}+ if (need_check) {+ i = min(i, i_max);+ }-template <cpy_kernel_t cpy_1>-static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,- const int nb12, const int nb13) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+ const block_q6_K * bxi = bx0 + i*blocks_per_row + kbxd;- if (i >= ne) {- return;+ x_dmf[i * (WARP_SIZE/QI6_K) + i / QI6_K + kbxd] = bxi->d;}- // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor- // then combine those indices with the corresponding byte offsets to get the total offsets- const int i03 = i/(ne00 * ne01 * ne02);- const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);- const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;- const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;- const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;-- const int i13 = i/(ne10 * ne11 * ne12);- const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);- const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;- const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;- const int dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;-- cpy_1(cx + x_offset, cdst + dst_offset);-}+#pragma unroll+ for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {+ int i = (i0 + i_offset * 8 + k / (WARP_SIZE/8)) % mmq_y;-static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q8_0 * dsti = (block_q8_0 *) cdsti;+ if (need_check) {+ i = min(i, i_max);+ }- float amax = 0.0f; // absolute max+ const block_q6_K * bxi = bx0 + i*blocks_per_row + (k % (WARP_SIZE/8)) / 4;- for (int j = 0; j < QK8_0; j++) {- const float v = xi[j];- amax = fmaxf(amax, fabsf(v));+ x_sc[i * (WARP_SIZE/8) + i / 8 + k % (WARP_SIZE/8)] = get_int_from_int8(bxi->scales, k % (QI6_K/8));}+}- const float d = amax / ((1 << 7) - 1);- const float id = d ? 1.0f/d : 0.0f;+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(+ const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,+ const int * __restrict__ y_qs, const half2 * __restrict__ y_ds, const int & i, const int & j, const int & k) {+ GGML_UNUSED(x_qh);- dsti->d = d;+ const float * x_dmf = (const float *) x_dm;+ const float * y_df = (const float *) y_ds;- for (int j = 0; j < QK8_0; ++j) {- const float x0 = xi[j]*id;+ const int8_t * sc = ((const int8_t *) &x_sc[i * (WARP_SIZE/8) + i/8 + k/8]);- dsti->qs[j] = roundf(x0);- }+ const int index_x = i * (QR6_K*WARP_SIZE + 1) + QR6_K*k;+ const int index_y = j * WARP_SIZE + (QR6_K*k) % WARP_SIZE;+ return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);}-static __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q4_0 * dsti = (block_q4_0 *) cdsti;-- float amax = 0.0f;- float vmax = 0.0f;+static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if QK_K == 256+ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;- for (int j = 0; j < QK4_0; ++j) {- const float v = xi[j];- if (amax < fabsf(v)) {- amax = fabsf(v);- vmax = v;+#if QR2_XXS == 8+ const int ib32 = iqs;+ const uint16_t * q2 = bq2->qs + 4*ib32;+ const uint8_t * aux8 = (const uint8_t *)q2;+ const int8_t * q8 = bq8_1[ib32].qs;+ uint32_t aux32 = q2[2] | (q2[3] << 16);+ int sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);+ const uint8_t signs = ksigns_iq2xs[aux32 & 127];+ for (int j = 0; j < 8; ++j) {+ sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);}+ q8 += 8;+ aux32 >>= 7;}-- const float d = vmax / -8;- const float id = d ? 1.0f/d : 0.0f;-- dsti->d = d;-- for (int j = 0; j < QK4_0/2; ++j) {- const float x0 = xi[0 + j]*id;- const float x1 = xi[QK4_0/2 + j]*id;-- const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));- const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));-- dsti->qs[j] = xi0;- dsti->qs[j] |= xi1 << 4;+ const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.25f;+ return d * sumi;+#else+ // iqs is 0...15+ const int ib32 = iqs/2;+ const int il = iqs%2;+ const uint16_t * q2 = bq2->qs + 4*ib32;+ const uint8_t * aux8 = (const uint8_t *)q2;+ const uint8_t * grid1 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+1]);+ const uint32_t aux32 = q2[2] | (q2[3] << 16);+ const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * __low2float(bq8_1[ib32].ds) * 0.25f;+ const uint8_t signs1 = ksigns_iq2xs[(aux32 >> 14*il) & 127];+ const uint8_t signs2 = ksigns_iq2xs[(aux32 >> (14*il + 7)) & 127];+ const int8_t * q8 = bq8_1[ib32].qs + 16*il;+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < 8; ++j) {+ sumi1 += q8[j+0] * grid1[j] * (signs1 & kmask_iq2xs[j] ? -1 : 1);+ sumi2 += q8[j+8] * grid2[j] * (signs2 & kmask_iq2xs[j] ? -1 : 1);}+ return d * (sumi1 + sumi2);+#endif+#else+ assert(false);+ return 0.f;+#endif}-static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q4_1 * dsti = (block_q4_1 *) cdsti;-- float vmin = FLT_MAX;- float vmax = -FLT_MAX;+static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;- for (int j = 0; j < QK4_1; ++j) {- const float v = xi[j];-- if (v < vmin) vmin = v;- if (v > vmax) vmax = v;+ const int ib32 = iqs;+ const uint16_t * q2 = bq2->qs + 4*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;+ const uint8_t ls2 = bq2->scales[ib32] >> 4;+ int sumi1 = 0;+ for (int l = 0; l < 2; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));+ const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);+ sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);+ sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);+ q8 += 8;}+ int sumi2 = 0;+ for (int l = 2; l < 4; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));+ const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);+ sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);+ sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);+ q8 += 8;+ }+ const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);+#else+ GGML_UNUSED(ksigns64);+ assert(false);+ return 0.f;+#endif+#else+ GGML_UNUSED(ksigns64);+ assert(false);+ return 0.f;+#endif+}- const float d = (vmax - vmin) / ((1 << 4) - 1);- const float id = d ? 1.0f/d : 0.0f;-- dsti->dm.x = d;- dsti->dm.y = vmin;+// TODO+static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq2_s * bq2 = (const block_iq2_s *) vbq;- for (int j = 0; j < QK4_1/2; ++j) {- const float x0 = (xi[0 + j] - vmin)*id;- const float x1 = (xi[QK4_1/2 + j] - vmin)*id;+ const int ib32 = iqs;+ const int8_t * q8 = bq8_1[ib32].qs;+ const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;+ const uint8_t ls2 = bq2->scales[ib32] >> 4;+ int sumi1 = 0;+ for (int l = 0; l < 2; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));+ const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);+ const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);+ const int grid_l = __vsub4(grid[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid[1] ^ signs1, signs1);+ sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);+ sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);+ q8 += 8;+ }+ int sumi2 = 0;+ for (int l = 2; l < 4; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));+ const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);+ const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);+ const int grid_l = __vsub4(grid[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid[1] ^ signs1, signs1);+ sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);+ sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);+ q8 += 8;+ }+ const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);+#else+ GGML_UNUSED(ksigns64);+ assert(false);+ return 0.f;+#endif+#else+ GGML_UNUSED(ksigns64);+ assert(false);+ return 0.f;+#endif+}- const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));- const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));+static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;- dsti->qs[j] = xi0;- dsti->qs[j] |= xi1 << 4;+ const int ib32 = iqs;+ const uint8_t * q3 = bq2->qs + 8*ib32;+ const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ uint32_t aux32 = gas[0] | (gas[1] << 16);+ int sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];+ const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));+ const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);+ sumi = __dp4a(grid_l, *((int *)q8+0), sumi);+ sumi = __dp4a(grid_h, *((int *)q8+1), sumi);+ q8 += 8;+ aux32 >>= 7;}+ const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;+ return d * sumi;+#else+ assert(false);+ return 0.f;+#endif+#else+ assert(false);+ return 0.f;+#endif}-template <cpy_kernel_t cpy_blck, int qk>-static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,- const int nb12, const int nb13) {- const int i = (blockDim.x*blockIdx.x + threadIdx.x)*qk;+// TODO: don't use lookup table for signs+static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq3_s * bq2 = (const block_iq3_s *) vbq;- if (i >= ne) {- return;+ const int ib32 = iqs;+ const uint8_t * qs = bq2->qs + 8*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ int sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));+ const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));+ uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);+ uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);+ const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);+ sumi = __dp4a(grid_l, *((int *)q8+0), sumi);+ sumi = __dp4a(grid_h, *((int *)q8+1), sumi);+ q8 += 8;}+ const float d = (float)bq2->d * (1 + 2*((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds);+ return d * sumi;+#else+ assert(false);+ return 0.f;+#endif+#else+ assert(false);+ return 0.f;+#endif+}- const int i03 = i/(ne00 * ne01 * ne02);- const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);- const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;- const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;- const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;-- const int i13 = i/(ne10 * ne11 * ne12);- const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);- const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;- const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;- const int dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;+static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if QK_K == 256+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq;- cpy_blck(cx + x_offset, cdst + dst_offset);+ const int ib32 = iqs;+ int sumi = 0;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const int * q8 = (const int *)bq8_1[ib32].qs;+ for (int l = 0; l < 4; ++l) {+ const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));+ int grid0 = grid[0] & 0x0f0f0f0f;+ int grid1 = (grid[0] >> 4) & 0x0f0f0f0f;+ sumi = __dp4a(q8[2*l+1], grid1, __dp4a(q8[2*l+0], grid0, sumi));+ }+#else+ const int8_t * q8 = bq8_1[ib32].qs;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));+ for (int j = 0; j < 4; ++j) {+ sumi += q8[j] * (grid[j] & 0xf) + q8[j+4] * (grid[j] >> 4);+ }+ q8 += 8;+ }+#endif+ const float delta = bq1->qh[ib32] & 0x8000 ? -1-IQ1S_DELTA : -1+IQ1S_DELTA;+ const float d1q = (float)bq1->d * (2*((bq1->qh[ib32] >> 12) & 7) + 1);+ const float d = d1q * __low2float (bq8_1[ib32].ds);+ const float m = d1q * __high2float(bq8_1[ib32].ds);+ return d * sumi + m * delta;+#else+ assert(false);+ return 0.f;+#endif}-static __device__ float rope_yarn_ramp(const float low, const float high, const int i0) {- const float y = (i0 / 2 - low) / max(0.001f, high - low);- return 1.0f - min(1.0f, max(0.0f, y));+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,+ int & val1, int & val2) {++ uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;+ aux32 = q4 & 0x0f0f0f0f;+ uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);+ uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);+ val1 = v1 | (v2 << 16);+ aux32 = (q4 >> 4) & 0x0f0f0f0f;+ v1 = values[q8[0]] | (values[q8[1]] << 8);+ v2 = values[q8[2]] | (values[q8[3]] << 8);+ val2 = v1 | (v2 << 16);}+#endif-struct rope_corr_dims {- float v[4];-};+static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn-// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.-static __device__ void rope_yarn(- float theta_extrap, float freq_scale, rope_corr_dims corr_dims, int64_t i0, float ext_factor, float mscale,- float * cos_theta, float * sin_theta-) {- // Get n-d rotational scaling corrected for extrapolation- float theta_interp = freq_scale * theta_extrap;- float theta = theta_interp;- if (ext_factor != 0.0f) {- float ramp_mix = rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;- theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;+ const block_iq4_nl * bq = (const block_iq4_nl *) vbq;- // Get n-d magnitude scaling corrected for interpolation- mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);- }- *cos_theta = cosf(theta) * mscale;- *sin_theta = sinf(theta) * mscale;-}+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;+ const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;-// rope == RoPE == rotary positional embedding-template<typename T, bool has_pos>-static __global__ void rope(- const T * x, T * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,- float ext_factor, float attn_factor, rope_corr_dims corr_dims-) {- const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;- if (col >= ncols) {- return;+ int v1, v2;+ int sumi1 = 0, sumi2 = 0;+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {+ const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);+ get_int_from_table_16(aux, values, v1, v2);+ sumi1 = __dp4a(v1, q8[l+0], sumi1);+ sumi2 = __dp4a(v2, q8[l+4], sumi2);}- const int row = blockDim.x*blockIdx.x + threadIdx.x;- const int i = row*ncols + col;- const int i2 = row/p_delta_rows;-- const int p = has_pos ? pos[i2] : 0;- const float theta_base = p*powf(freq_base, -float(col)/ncols);-- float cos_theta, sin_theta;- rope_yarn(theta_base, freq_scale, corr_dims, col, ext_factor, attn_factor, &cos_theta, &sin_theta);-- const float x0 = x[i + 0];- const float x1 = x[i + 1];+#else+ const uint8_t * q4 = bq->qs + 4*iqs;+ const int8_t * q8 = bq8_1->qs + 4*iqs;- dst[i + 0] = x0*cos_theta - x1*sin_theta;- dst[i + 1] = x0*sin_theta + x1*cos_theta;+ int sumi1 = 0, sumi2 = 0;+ for (int l = 0; l < 4*VDR_Q4_0_Q8_1_MMVQ; ++l) {+ sumi1 += q8[l+ 0] * kvalues_iq4nl[q4[l] & 0xf];+ sumi2 += q8[l+16] * kvalues_iq4nl[q4[l] >> 4];+ }+#endif+ const float d = (float)bq->d * __low2float(bq8_1->ds);+ return d * (sumi1 + sumi2);}-template<typename T, bool has_pos>-static __global__ void rope_neox(- const T * x, T * dst, int ncols, int n_dims, const int32_t * pos, float freq_scale, int p_delta_rows,- float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, float inv_ndims-) {- const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);-- if (col >= ncols) {- return;- }+static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- const int row = blockDim.x*blockIdx.x + threadIdx.x;- const int ib = col / n_dims;- const int ic = col % n_dims;+#if QK_K == 256+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- if (ib > 0) {- const int i = row*ncols + ib*n_dims + ic;+ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;- dst[i + 0] = x[i + 0];- dst[i + 1] = x[i + 1];+ //// iqs is 0...7+ //const int ib64 = iqs/2;+ //const int il = iqs%2;+ //const int32_t * q8_1 = (const int *)bq8_1[2*ib64+0].qs + 2*il;+ //const int32_t * q8_2 = (const int *)bq8_1[2*ib64+1].qs + 2*il;+ //const uint32_t * q4_1 = (const uint32_t *)bq4->qs + 8*ib64 + 2*il;+ //const uint32_t * q4_2 = q4_1 + 4;+ //const int8_t ls1 = (bq4->scales_l[ib64] & 0xf) | (((bq4->scales_h >> (4*ib64+0)) & 3) << 4);+ //const int8_t ls2 = (bq4->scales_l[ib64] >> 4) | (((bq4->scales_h >> (4*ib64+2)) & 3) << 4);+ //const float d1 = (float)bq4->d * (ls1 - 32) * __low2float(bq8_1[2*ib64+0].ds);+ //const float d2 = (float)bq4->d * (ls2 - 32) * __low2float(bq8_1[2*ib64+1].ds);+ //int v1, v2;+ //int sumi1 = 0, sumi2 = 0;+ //for (int j = 0; j < 2; ++j) {+ // get_int_from_table_16(q4_1[j], values, v1, v2);+ // sumi1 = __dp4a(v2, q8_1[j+4], __dp4a(v1, q8_1[j+0], sumi1));+ // get_int_from_table_16(q4_2[j], values, v1, v2);+ // sumi2 = __dp4a(v2, q8_2[j+4], __dp4a(v1, q8_2[j+0], sumi2));+ //}+ //return d1 * sumi1 + d2 * sumi2;- return;+ // iqs is 0...7+ const int ib32 = iqs;+ const int32_t * q8 = (const int *)bq8_1[ib32].qs;+ const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;+ const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);+ const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds);+ int v1, v2;+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < 4; ++j) {+ get_int_from_table_16(q4[j], values, v1, v2);+ sumi1 = __dp4a(v1, q8[j+0], sumi1);+ sumi2 = __dp4a(v2, q8[j+4], sumi2);}+ return d * (sumi1 + sumi2);- const int i = row*ncols + ib*n_dims + ic/2;- const int i2 = row/p_delta_rows;+ //// iqs is 0...15+ //const int ib32 = iqs/2;+ //const int il = iqs%2;+ //const int32_t * q8 = (const int *)bq8_1[ib32].qs + 2*il;+ //const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32 + 2*il;+ //const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);+ //const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds);+ //int v1, v2;+ //int sumi1 = 0, sumi2 = 0;+ //for (int j = 0; j < 2; ++j) {+ // get_int_from_table_16(q4[j], values, v1, v2);+ // sumi1 = __dp4a(v1, q8[j+0], sumi1);+ // sumi2 = __dp4a(v2, q8[j+4], sumi2);+ //}+ //return d * (sumi1 + sumi2);+#else+ assert(false);+ return 0.f;+#endif+#else+ return vec_dot_iq4_xs_q8_1(vbq, bq8_1, iqs);+#endif+}- float cur_rot = inv_ndims * ic - ib;+template <int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,+ allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>+static __device__ __forceinline__ void mul_mat_q(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const int p = has_pos ? pos[i2] : 0;- const float theta_base = p*freq_scale*powf(theta_scale, col/2.0f);+ const block_q_t * x = (const block_q_t *) vx;+ const block_q8_1 * y = (const block_q8_1 *) vy;- float cos_theta, sin_theta;- rope_yarn(theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);+ const int blocks_per_row_x = ncols_x / qk;+ const int blocks_per_col_y = nrows_y / QK8_1;+ const int blocks_per_warp = WARP_SIZE / qi;- const float x0 = x[i + 0];- const float x1 = x[i + n_dims/2];+ const int & ncols_dst = ncols_y;- dst[i + 0] = x0*cos_theta - x1*sin_theta;- dst[i + n_dims/2] = x0*sin_theta + x1*cos_theta;-}+ const int row_dst_0 = blockIdx.x*mmq_y;+ const int & row_x_0 = row_dst_0;-static __global__ void rope_glm_f32(- const float * x, float * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,- int n_ctx-) {- const int col = blockDim.x*blockIdx.x + threadIdx.x;- const int half_n_dims = ncols/4;+ const int col_dst_0 = blockIdx.y*mmq_x;+ const int & col_y_0 = col_dst_0;- if (col >= half_n_dims) {- return;- }+ int * tile_x_ql = nullptr;+ half2 * tile_x_dm = nullptr;+ int * tile_x_qh = nullptr;+ int * tile_x_sc = nullptr;- const int row = blockDim.y*blockIdx.y + threadIdx.y;- const int i = row*ncols + col;- const int i2 = row/p_delta_rows;+ allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);- const float col_theta_scale = powf(freq_base, -2.0f*col/ncols);- // FIXME: this is likely wrong- const int p = pos != nullptr ? pos[i2] : 0;+ __shared__ int tile_y_qs[mmq_x * WARP_SIZE];+ __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE/QI8_1];- const float theta = min(p, n_ctx - 2)*freq_scale*col_theta_scale;- const float sin_theta = sinf(theta);- const float cos_theta = cosf(theta);+ float sum[mmq_y/WARP_SIZE][mmq_x/nwarps] = {{0.0f}};- const float x0 = x[i + 0];- const float x1 = x[i + half_n_dims];+ for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {- dst[i + 0] = x0*cos_theta - x1*sin_theta;- dst[i + half_n_dims] = x0*sin_theta + x1*cos_theta;+ load_tiles(x + row_x_0*blocks_per_row_x + ib0, tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc,+ threadIdx.y, nrows_x-row_x_0-1, threadIdx.x, blocks_per_row_x);- const float block_theta = ((float)max(p - n_ctx - 2, 0))*col_theta_scale;- const float sin_block_theta = sinf(block_theta);- const float cos_block_theta = cosf(block_theta);+#pragma unroll+ for (int ir = 0; ir < qr; ++ir) {+ const int kqs = ir*WARP_SIZE + threadIdx.x;+ const int kbxd = kqs / QI8_1;- const float x2 = x[i + half_n_dims * 2];- const float x3 = x[i + half_n_dims * 3];+#pragma unroll+ for (int i = 0; i < mmq_x; i += nwarps) {+ const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses- dst[i + half_n_dims * 2] = x2*cos_block_theta - x3*sin_block_theta;- dst[i + half_n_dims * 3] = x2*sin_block_theta + x3*cos_block_theta;-}+ const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];-static __global__ void alibi_f32(const float * x, float * dst, const int ncols, const int k_rows,- const int n_heads_log2_floor, const float m0, const float m1) {- const int col = blockDim.x*blockIdx.x + threadIdx.x;+ const int index_y = (threadIdx.y + i) * WARP_SIZE + kqs % WARP_SIZE;+ tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);+ }- if (col >= ncols) {- return;- }+#pragma unroll+ for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {+ const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE/QI8_1)) % mmq_x;+ const int kby = threadIdx.x % (WARP_SIZE/QI8_1);+ const int col_y_eff = min(col_y_0 + ids, ncols_y-1);- const int row = blockDim.y*blockIdx.y + threadIdx.y;- const int i = row*ncols + col;+ // if the sum is not needed it's faster to transform the scale to f32 ahead of time+ const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE/QI8_1) + kby].ds;+ half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE/QI8_1) + kby];+ if (need_sum) {+ *dsi_dst = *dsi_src;+ } else {+ float * dfi_dst = (float *) dsi_dst;+ *dfi_dst = __low2float(*dsi_src);+ }+ }- const int k = row/k_rows;+ __syncthreads();- float m_k;- if (k < n_heads_log2_floor) {- m_k = powf(m0, k + 1);- } else {- m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);+// #pragma unroll // unrolling this loop causes too much register pressure+ for (int k = ir*WARP_SIZE/qr; k < (ir+1)*WARP_SIZE/qr; k += vdr) {+#pragma unroll+ for (int j = 0; j < mmq_x; j += nwarps) {+#pragma unroll+ for (int i = 0; i < mmq_y; i += WARP_SIZE) {+ sum[i/WARP_SIZE][j/nwarps] += vec_dot(+ tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,+ threadIdx.x + i, threadIdx.y + j, k);+ }+ }+ }++ __syncthreads();+ }}- dst[i] = col * m_k + x[i];-}+#pragma unroll+ for (int j = 0; j < mmq_x; j += nwarps) {+ const int col_dst = col_dst_0 + j + threadIdx.y;-static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {- const int row = blockIdx.x;- const int col = threadIdx.x;+ if (col_dst >= ncols_dst) {+ return;+ }- float sum = 0.0f;- for (int i = col; i < ncols; i += blockDim.x) {- sum += x[row * ncols + i];- }+#pragma unroll+ for (int i = 0; i < mmq_y; i += WARP_SIZE) {+ const int row_dst = row_dst_0 + threadIdx.x + i;- sum = warp_reduce_sum(sum);+ if (row_dst >= nrows_dst) {+ continue;+ }- if (col == 0) {- dst[row] = sum;+ dst[col_dst*nrows_dst + row_dst] = sum[i/WARP_SIZE][j/nwarps];+ }}}-template<typename T>-static inline __device__ void swap(T & a, T & b) {- T tmp = a;- a = b;- b = tmp;-}--template<ggml_sort_order order>-static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols) {- // bitonic sort- int col = threadIdx.x;- int row = blockIdx.y;+#define MMQ_X_Q4_0_RDNA2 64+#define MMQ_Y_Q4_0_RDNA2 128+#define NWARPS_Q4_0_RDNA2 8+#define MMQ_X_Q4_0_RDNA1 64+#define MMQ_Y_Q4_0_RDNA1 64+#define NWARPS_Q4_0_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q4_0_AMPERE 4+#define MMQ_Y_Q4_0_AMPERE 32+#define NWARPS_Q4_0_AMPERE 4+#else+#define MMQ_X_Q4_0_AMPERE 64+#define MMQ_Y_Q4_0_AMPERE 128+#define NWARPS_Q4_0_AMPERE 4+#endif+#define MMQ_X_Q4_0_PASCAL 64+#define MMQ_Y_Q4_0_PASCAL 64+#define NWARPS_Q4_0_PASCAL 8- if (col >= ncols) return;+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q4_0_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ mul_mat_q4_0(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const float * x_row = x + row * ncols;- int * dst_row = dst + row * ncols;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q4_0_RDNA2;+ const int mmq_y = MMQ_Y_Q4_0_RDNA2;+ const int nwarps = NWARPS_Q4_0_RDNA2;+#else+ const int mmq_x = MMQ_X_Q4_0_RDNA1;+ const int mmq_y = MMQ_Y_Q4_0_RDNA1;+ const int nwarps = NWARPS_Q4_0_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- // initialize indices- if (col < ncols) {- dst_row[col] = col;- }- __syncthreads();+ mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,+ load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- for (int k = 2; k <= ncols; k *= 2) {- for (int j = k / 2; j > 0; j /= 2) {- int ixj = col ^ j;- if (ixj > col) {- if ((col & k) == 0) {- if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) {- swap(dst_row[col], dst_row[ixj]);- }- } else {- if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) {- swap(dst_row[col], dst_row[ixj]);- }- }- }- __syncthreads();- }- }-}+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q4_0_AMPERE;+ const int mmq_y = MMQ_Y_Q4_0_AMPERE;+ const int nwarps = NWARPS_Q4_0_AMPERE;-static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {- const int col = blockDim.y*blockIdx.y + threadIdx.y;- const int row = blockDim.x*blockIdx.x + threadIdx.x;+ mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,+ load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- if (col >= ncols) {- return;- }+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q4_0_PASCAL;+ const int mmq_y = MMQ_Y_Q4_0_PASCAL;+ const int nwarps = NWARPS_Q4_0_PASCAL;- const int i = row*ncols + col;- //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];- //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU- dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;+ mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,+ load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q4_0_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA}-template <bool vals_smem, int ncols_template, int block_size_template>-static __global__ void soft_max_f32(const float * x, const float * mask, const float * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {- const int ncols = ncols_template == 0 ? ncols_par : ncols_template;+#define MMQ_X_Q4_1_RDNA2 64+#define MMQ_Y_Q4_1_RDNA2 128+#define NWARPS_Q4_1_RDNA2 8+#define MMQ_X_Q4_1_RDNA1 64+#define MMQ_Y_Q4_1_RDNA1 64+#define NWARPS_Q4_1_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q4_1_AMPERE 4+#define MMQ_Y_Q4_1_AMPERE 32+#define NWARPS_Q4_1_AMPERE 4+#else+#define MMQ_X_Q4_1_AMPERE 64+#define MMQ_Y_Q4_1_AMPERE 128+#define NWARPS_Q4_1_AMPERE 4+#endif+#define MMQ_X_Q4_1_PASCAL 64+#define MMQ_Y_Q4_1_PASCAL 64+#define NWARPS_Q4_1_PASCAL 8- const int tid = threadIdx.x;- const int rowx = blockIdx.x;- const int rowy = rowx % nrows_y; // broadcast the mask in the row dimension+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#elif __CUDA_ARCH__ < CC_VOLTA+ __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_PASCAL, 2)+#endif // __CUDA_ARCH__ < CC_VOLTA+ mul_mat_q4_1(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const int block_size = block_size_template == 0 ? blockDim.x : block_size_template;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q4_1_RDNA2;+ const int mmq_y = MMQ_Y_Q4_1_RDNA2;+ const int nwarps = NWARPS_Q4_1_RDNA2;+#else+ const int mmq_x = MMQ_X_Q4_1_RDNA1;+ const int mmq_y = MMQ_Y_Q4_1_RDNA1;+ const int nwarps = NWARPS_Q4_1_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- const int warp_id = threadIdx.x / WARP_SIZE;- const int lane_id = threadIdx.x % WARP_SIZE;+ mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,+ load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- float slope = 0.0f;+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q4_1_AMPERE;+ const int mmq_y = MMQ_Y_Q4_1_AMPERE;+ const int nwarps = NWARPS_Q4_1_AMPERE;- // ALiBi- if (max_bias > 0.0f) {- const int h = rowx/nrows_y; // head index+ mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,+ load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- const float base = h < n_head_log2 ? m0 : m1;- const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q4_1_PASCAL;+ const int mmq_y = MMQ_Y_Q4_1_PASCAL;+ const int nwarps = NWARPS_Q4_1_PASCAL;- slope = powf(base, exp);- }+ mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,+ load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q4_1_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA+}- extern __shared__ float data_soft_max_f32[];- float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication- // shared memory buffer to cache values between iterations:- float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + rowx*ncols;+#define MMQ_X_Q5_0_RDNA2 64+#define MMQ_Y_Q5_0_RDNA2 128+#define NWARPS_Q5_0_RDNA2 8+#define MMQ_X_Q5_0_RDNA1 64+#define MMQ_Y_Q5_0_RDNA1 64+#define NWARPS_Q5_0_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q5_0_AMPERE 4+#define MMQ_Y_Q5_0_AMPERE 32+#define NWARPS_Q5_0_AMPERE 4+#else+#define MMQ_X_Q5_0_AMPERE 128+#define MMQ_Y_Q5_0_AMPERE 64+#define NWARPS_Q5_0_AMPERE 4+#endif+#define MMQ_X_Q5_0_PASCAL 64+#define MMQ_Y_Q5_0_PASCAL 64+#define NWARPS_Q5_0_PASCAL 8- float max_val = -INFINITY;+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q5_0_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ mul_mat_q5_0(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#pragma unroll- for (int col0 = 0; col0 < ncols; col0 += block_size) {- const int col = col0 + tid;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q5_0_RDNA2;+ const int mmq_y = MMQ_Y_Q5_0_RDNA2;+ const int nwarps = NWARPS_Q5_0_RDNA2;+#else+ const int mmq_x = MMQ_X_Q5_0_RDNA1;+ const int mmq_y = MMQ_Y_Q5_0_RDNA1;+ const int nwarps = NWARPS_Q5_0_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- if (ncols_template == 0 && col >= ncols) {- break;- }+ mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,+ load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- const int ix = rowx*ncols + col;- const int iy = rowy*ncols + col;+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q5_0_AMPERE;+ const int mmq_y = MMQ_Y_Q5_0_AMPERE;+ const int nwarps = NWARPS_Q5_0_AMPERE;- const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + (pos ? slope*pos[col] : 0.0f);+ mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,+ load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- vals[col] = val;- max_val = max(max_val, val);- }+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q5_0_PASCAL;+ const int mmq_y = MMQ_Y_Q5_0_PASCAL;+ const int nwarps = NWARPS_Q5_0_PASCAL;- // find the max value in the block- max_val = warp_reduce_max(max_val);- if (block_size > WARP_SIZE) {- if (warp_id == 0) {- buf_iw[lane_id] = -INFINITY;- }- __syncthreads();+ mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,+ load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q5_0_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA+}- if (lane_id == 0) {- buf_iw[warp_id] = max_val;- }- __syncthreads();+#define MMQ_X_Q5_1_RDNA2 64+#define MMQ_Y_Q5_1_RDNA2 128+#define NWARPS_Q5_1_RDNA2 8+#define MMQ_X_Q5_1_RDNA1 64+#define MMQ_Y_Q5_1_RDNA1 64+#define NWARPS_Q5_1_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q5_1_AMPERE 4+#define MMQ_Y_Q5_1_AMPERE 32+#define NWARPS_Q5_1_AMPERE 4+#else+#define MMQ_X_Q5_1_AMPERE 128+#define MMQ_Y_Q5_1_AMPERE 64+#define NWARPS_Q5_1_AMPERE 4+#endif+#define MMQ_X_Q5_1_PASCAL 64+#define MMQ_Y_Q5_1_PASCAL 64+#define NWARPS_Q5_1_PASCAL 8- max_val = buf_iw[lane_id];- max_val = warp_reduce_max(max_val);- }+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q5_1_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+mul_mat_q5_1(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- float tmp = 0.0f; // partial sum+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q5_1_RDNA2;+ const int mmq_y = MMQ_Y_Q5_1_RDNA2;+ const int nwarps = NWARPS_Q5_1_RDNA2;+#else+ const int mmq_x = MMQ_X_Q5_1_RDNA1;+ const int mmq_y = MMQ_Y_Q5_1_RDNA1;+ const int nwarps = NWARPS_Q5_1_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)-#pragma unroll- for (int col0 = 0; col0 < ncols; col0 += block_size) {- const int col = col0 + tid;+ mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,+ load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- if (ncols_template == 0 && col >= ncols) {- break;- }+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q5_1_AMPERE;+ const int mmq_y = MMQ_Y_Q5_1_AMPERE;+ const int nwarps = NWARPS_Q5_1_AMPERE;- const float val = expf(vals[col] - max_val);- tmp += val;- vals[col] = val;- }+ mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,+ load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- // find the sum of exps in the block- tmp = warp_reduce_sum(tmp);- if (block_size > WARP_SIZE) {- if (warp_id == 0) {- buf_iw[lane_id] = 0.0f;- }- __syncthreads();+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q5_1_PASCAL;+ const int mmq_y = MMQ_Y_Q5_1_PASCAL;+ const int nwarps = NWARPS_Q5_1_PASCAL;- if (lane_id == 0) {- buf_iw[warp_id] = tmp;- }- __syncthreads();+ mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,+ load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q5_1_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA+}- tmp = buf_iw[lane_id];- tmp = warp_reduce_sum(tmp);- }+#define MMQ_X_Q8_0_RDNA2 64+#define MMQ_Y_Q8_0_RDNA2 128+#define NWARPS_Q8_0_RDNA2 8+#define MMQ_X_Q8_0_RDNA1 64+#define MMQ_Y_Q8_0_RDNA1 64+#define NWARPS_Q8_0_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q8_0_AMPERE 4+#define MMQ_Y_Q8_0_AMPERE 32+#define NWARPS_Q8_0_AMPERE 4+#else+#define MMQ_X_Q8_0_AMPERE 128+#define MMQ_Y_Q8_0_AMPERE 64+#define NWARPS_Q8_0_AMPERE 4+#endif+#define MMQ_X_Q8_0_PASCAL 64+#define MMQ_Y_Q8_0_PASCAL 64+#define NWARPS_Q8_0_PASCAL 8- const float inv_sum = 1.0f / tmp;+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q8_0_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ mul_mat_q8_0(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#pragma unroll- for (int col0 = 0; col0 < ncols; col0 += block_size) {- const int col = col0 + tid;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q8_0_RDNA2;+ const int mmq_y = MMQ_Y_Q8_0_RDNA2;+ const int nwarps = NWARPS_Q8_0_RDNA2;+#else+ const int mmq_x = MMQ_X_Q8_0_RDNA1;+ const int mmq_y = MMQ_Y_Q8_0_RDNA1;+ const int nwarps = NWARPS_Q8_0_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- if (ncols_template == 0 && col >= ncols) {- return;- }+ mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,+ load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- const int idst = rowx*ncols + col;- dst[idst] = vals[col] * inv_sum;- }-}+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q8_0_AMPERE;+ const int mmq_y = MMQ_Y_Q8_0_AMPERE;+ const int nwarps = NWARPS_Q8_0_AMPERE;-static __global__ void scale_f32(const float * x, float * dst, const float scale, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+ mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,+ load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- if (i >= k) {- return;- }+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q8_0_PASCAL;+ const int mmq_y = MMQ_Y_Q8_0_PASCAL;+ const int nwarps = NWARPS_Q8_0_PASCAL;- dst[i] = scale * x[i];+ mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,+ load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q8_0_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA}-static __global__ void clamp_f32(const float * x, float * dst, const float min, const float max, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;-- if (i >= k) {- return;- }+#define MMQ_X_Q2_K_RDNA2 64+#define MMQ_Y_Q2_K_RDNA2 128+#define NWARPS_Q2_K_RDNA2 8+#define MMQ_X_Q2_K_RDNA1 128+#define MMQ_Y_Q2_K_RDNA1 32+#define NWARPS_Q2_K_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q2_K_AMPERE 4+#define MMQ_Y_Q2_K_AMPERE 32+#define NWARPS_Q2_K_AMPERE 4+#else+#define MMQ_X_Q2_K_AMPERE 64+#define MMQ_Y_Q2_K_AMPERE 128+#define NWARPS_Q2_K_AMPERE 4+#endif+#define MMQ_X_Q2_K_PASCAL 64+#define MMQ_Y_Q2_K_PASCAL 64+#define NWARPS_Q2_K_PASCAL 8- dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]);-}+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q2_K_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+mul_mat_q2_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-template <typename T>-static __global__ void im2col_kernel(- const float * x, T * dst, int batch_offset,- int offset_delta, int IC, int IW, int IH, int OH, int OW, int KW, int KH, int pelements, int CHW,- int s0, int s1, int p0, int p1, int d0, int d1) {- const int i = threadIdx.x + blockIdx.x * blockDim.x;- if (i >= pelements) {- return;- }+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q2_K_RDNA2;+ const int mmq_y = MMQ_Y_Q2_K_RDNA2;+ const int nwarps = NWARPS_Q2_K_RDNA2;+#else+ const int mmq_x = MMQ_X_Q2_K_RDNA1;+ const int mmq_y = MMQ_Y_Q2_K_RDNA1;+ const int nwarps = NWARPS_Q2_K_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- const int ksize = OW * (KH > 1 ? KW : 1);- const int kx = i / ksize;- const int kd = kx * ksize;- const int ky = (i - kd) / OW;- const int ix = i % OW;+ mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,+ load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- const int oh = blockIdx.y;- const int batch = blockIdx.z / IC;- const int ic = blockIdx.z % IC;+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q2_K_AMPERE;+ const int mmq_y = MMQ_Y_Q2_K_AMPERE;+ const int nwarps = NWARPS_Q2_K_AMPERE;- const int64_t iiw = ix * s0 + kx * d0 - p0;- const int64_t iih = oh * s1 + ky * d1 - p1;+ mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,+ load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- const int64_t offset_dst =- ((batch * OH + oh) * OW + ix) * CHW +- (ic * (KW * KH) + ky * KW + kx);+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q2_K_PASCAL;+ const int mmq_y = MMQ_Y_Q2_K_PASCAL;+ const int nwarps = NWARPS_Q2_K_PASCAL;- if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {- dst[offset_dst] = 0.0f;- } else {- const int64_t offset_src = ic * offset_delta + batch * batch_offset;- dst[offset_dst] = x[offset_src + iih * IW + iiw];- }+ mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,+ load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q2_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA}-template <typename Ti, typename To>-static __global__ void pool2d_nchw_kernel(- const int ih, const int iw, const int oh, const int ow,- const int kh, const int kw, const int sh, const int sw,- const int ph, const int pw, const int parallel_elements,- const Ti* src, To* dst, const enum ggml_op_pool op) {- int idx = threadIdx.x + blockIdx.x * blockDim.x;- if (idx >= parallel_elements) {- return;- }-- const int I_HW = ih * iw;- const int O_HW = oh * ow;- const int nc = idx / O_HW;- const int cur_oh = idx % O_HW / ow;- const int cur_ow = idx % O_HW % ow;- const Ti* i_ptr = src + nc * I_HW;- To* o_ptr = dst + nc * O_HW;- const int start_h = cur_oh * sh - ph;- const int bh = max(0, start_h);- const int eh = min(ih, start_h + kh);- const int start_w = cur_ow * sw - pw;- const int bw = max(0, start_w);- const int ew = min(iw, start_w + kw);- const To scale = 1. / (kh * kw);- To res = 0;-- switch (op) {- case GGML_OP_POOL_AVG: res = 0; break;- case GGML_OP_POOL_MAX: res = -FLT_MAX; break;- }-- for (int i = bh; i < eh; i += 1) {- for (int j = bw; j < ew; j += 1) {- #if __CUDA_ARCH__ >= 350- Ti cur = __ldg(i_ptr + i * iw + j);- #else- Ti cur = i_ptr[i * iw + j];- #endif- switch (op) {- case GGML_OP_POOL_AVG: res += cur * scale; break;- case GGML_OP_POOL_MAX: res = max(res, (To)cur); break;- }- }- }- o_ptr[cur_oh * ow + cur_ow] = res;-}--template<int qk, int qr, dequantize_kernel_t dq>-static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {+#define MMQ_X_Q3_K_RDNA2 128+#define MMQ_Y_Q3_K_RDNA2 64+#define NWARPS_Q3_K_RDNA2 8+#define MMQ_X_Q3_K_RDNA1 32+#define MMQ_Y_Q3_K_RDNA1 128+#define NWARPS_Q3_K_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q3_K_AMPERE 4+#define MMQ_Y_Q3_K_AMPERE 32+#define NWARPS_Q3_K_AMPERE 4+#else+#define MMQ_X_Q3_K_AMPERE 128+#define MMQ_Y_Q3_K_AMPERE 128+#define NWARPS_Q3_K_AMPERE 4+#endif+#define MMQ_X_Q3_K_PASCAL 64+#define MMQ_Y_Q3_K_PASCAL 64+#define NWARPS_Q3_K_PASCAL 8- GGML_TENSOR_BINARY_OP_LOCALS+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#elif __CUDA_ARCH__ < CC_VOLTA+ __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_PASCAL, 2)+#endif // __CUDA_ARCH__ < CC_VOLTA+ mul_mat_q3_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);- const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);- const dim3 block_nums(block_num_x, ne10, ne11*ne12);+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q3_K_RDNA2;+ const int mmq_y = MMQ_Y_Q3_K_RDNA2;+ const int nwarps = NWARPS_Q3_K_RDNA2;+#else+ const int mmq_x = MMQ_X_Q3_K_RDNA1;+ const int mmq_y = MMQ_Y_Q3_K_RDNA1;+ const int nwarps = NWARPS_Q3_K_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- // strides in elements- //const size_t s0 = nb0 / ggml_element_size(dst);- const size_t s1 = nb1 / ggml_element_size(dst);- const size_t s2 = nb2 / ggml_element_size(dst);- const size_t s3 = nb3 / ggml_element_size(dst);+ mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,+ load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- const size_t s10 = nb10 / ggml_element_size(src1);- const size_t s11 = nb11 / ggml_element_size(src1);- const size_t s12 = nb12 / ggml_element_size(src1);- //const size_t s13 = nb13 / ggml_element_size(src1);+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q3_K_AMPERE;+ const int mmq_y = MMQ_Y_Q3_K_AMPERE;+ const int nwarps = NWARPS_Q3_K_AMPERE;- GGML_ASSERT(ne00 % 2 == 0);+ mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,+ load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne00, /*ne01, ne02, ne03,*/- /*ne10, ne11,*/ ne12, /*ne13,*/- /* s0,*/ s1, s2, s3,- /* nb00,*/ nb01, nb02, nb03,- s10, s11, s12/*, s13*/);+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q3_K_PASCAL;+ const int mmq_y = MMQ_Y_Q3_K_PASCAL;+ const int nwarps = NWARPS_Q3_K_PASCAL;- (void) dst;+ mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,+ load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q3_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA}-template<typename src0_t>-static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {+#define MMQ_X_Q4_K_RDNA2 64+#define MMQ_Y_Q4_K_RDNA2 128+#define NWARPS_Q4_K_RDNA2 8+#define MMQ_X_Q4_K_RDNA1 32+#define MMQ_Y_Q4_K_RDNA1 64+#define NWARPS_Q4_K_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q4_K_AMPERE 4+#define MMQ_Y_Q4_K_AMPERE 32+#define NWARPS_Q4_K_AMPERE 4+#else+#define MMQ_X_Q4_K_AMPERE 64+#define MMQ_Y_Q4_K_AMPERE 128+#define NWARPS_Q4_K_AMPERE 4+#endif+#define MMQ_X_Q4_K_PASCAL 64+#define MMQ_Y_Q4_K_PASCAL 64+#define NWARPS_Q4_K_PASCAL 8- GGML_TENSOR_BINARY_OP_LOCALS+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#elif __CUDA_ARCH__ < CC_VOLTA+ __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_PASCAL, 2)+#endif // __CUDA_ARCH__ < CC_VOLTA+ mul_mat_q4_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);- const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;- const dim3 block_nums(block_num_x, ne10, ne11*ne12);+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q4_K_RDNA2;+ const int mmq_y = MMQ_Y_Q4_K_RDNA2;+ const int nwarps = NWARPS_Q4_K_RDNA2;+#else+ const int mmq_x = MMQ_X_Q4_K_RDNA1;+ const int mmq_y = MMQ_Y_Q4_K_RDNA1;+ const int nwarps = NWARPS_Q4_K_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- // strides in elements- //const size_t s0 = nb0 / ggml_element_size(dst);- const size_t s1 = nb1 / ggml_element_size(dst);- const size_t s2 = nb2 / ggml_element_size(dst);- const size_t s3 = nb3 / ggml_element_size(dst);+ mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,+ load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- const size_t s10 = nb10 / ggml_element_size(src1);- const size_t s11 = nb11 / ggml_element_size(src1);- const size_t s12 = nb12 / ggml_element_size(src1);- //const size_t s13 = nb13 / ggml_element_size(src1);+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q4_K_AMPERE;+ const int mmq_y = MMQ_Y_Q4_K_AMPERE;+ const int nwarps = NWARPS_Q4_K_AMPERE;- k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne00, /*ne01, ne02, ne03,*/- /*ne10, ne11,*/ ne12, /*ne13,*/- /* s0,*/ s1, s2, s3,- /* nb00,*/ nb01, nb02, nb03,- s10, s11, s12/*, s13*/);+ mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,+ load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);++#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q4_K_PASCAL;+ const int mmq_y = MMQ_Y_Q4_K_PASCAL;+ const int nwarps = NWARPS_Q4_K_PASCAL;- (void) dst;+ mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,+ load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q4_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA}-template<float (*bin_op)(const float, const float)>-struct bin_bcast_cuda {- template<typename src0_t, typename src1_t, typename dst_t>- void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,- const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,- cudaStream_t stream) {+#define MMQ_X_Q5_K_RDNA2 64+#define MMQ_Y_Q5_K_RDNA2 128+#define NWARPS_Q5_K_RDNA2 8+#define MMQ_X_Q5_K_RDNA1 32+#define MMQ_Y_Q5_K_RDNA1 64+#define NWARPS_Q5_K_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q5_K_AMPERE 4+#define MMQ_Y_Q5_K_AMPERE 32+#define NWARPS_Q5_K_AMPERE 4+#else+#define MMQ_X_Q5_K_AMPERE 64+#define MMQ_Y_Q5_K_AMPERE 128+#define NWARPS_Q5_K_AMPERE 4+#endif+#define MMQ_X_Q5_K_PASCAL 64+#define MMQ_Y_Q5_K_PASCAL 64+#define NWARPS_Q5_K_PASCAL 8- GGML_TENSOR_BINARY_OP_LOCALS+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q5_K_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+mul_mat_q5_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- int nr0 = ne10/ne0;- int nr1 = ne11/ne1;- int nr2 = ne12/ne2;- int nr3 = ne13/ne3;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q5_K_RDNA2;+ const int mmq_y = MMQ_Y_Q5_K_RDNA2;+ const int nwarps = NWARPS_Q5_K_RDNA2;+#else+ const int mmq_x = MMQ_X_Q5_K_RDNA1;+ const int mmq_y = MMQ_Y_Q5_K_RDNA1;+ const int nwarps = NWARPS_Q5_K_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- int nr[4] = { nr0, nr1, nr2, nr3 };+ mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,+ load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- // collapse dimensions until first broadcast dimension- int64_t cne0[] = {ne0, ne1, ne2, ne3};- int64_t cne1[] = {ne10, ne11, ne12, ne13};- size_t cnb0[] = {nb0, nb1, nb2, nb3};- size_t cnb1[] = {nb10, nb11, nb12, nb13};- auto collapse = [](int64_t cne[]) {- cne[0] *= cne[1];- cne[1] = cne[2];- cne[2] = cne[3];- cne[3] = 1;- };+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q5_K_AMPERE;+ const int mmq_y = MMQ_Y_Q5_K_AMPERE;+ const int nwarps = NWARPS_Q5_K_AMPERE;- auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {- cnb[1] *= cne[1];- cnb[2] *= cne[2];- cnb[3] *= cne[3];- };+ mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,+ load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- for (int i = 0; i < 4; i++) {- if (nr[i] != 1) {- break;- }- if (i > 0) {- collapse_nb(cnb0, cne0);- collapse_nb(cnb1, cne1);- collapse(cne0);- collapse(cne1);- }- }- {- int64_t ne0 = cne0[0];- int64_t ne1 = cne0[1];- int64_t ne2 = cne0[2];- int64_t ne3 = cne0[3];+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q5_K_PASCAL;+ const int mmq_y = MMQ_Y_Q5_K_PASCAL;+ const int nwarps = NWARPS_Q5_K_PASCAL;- int64_t ne10 = cne1[0];- int64_t ne11 = cne1[1];- int64_t ne12 = cne1[2];- int64_t ne13 = cne1[3];+ mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,+ load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q5_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA+}- size_t nb0 = cnb0[0];- size_t nb1 = cnb0[1];- size_t nb2 = cnb0[2];- size_t nb3 = cnb0[3];+#define MMQ_X_Q6_K_RDNA2 64+#define MMQ_Y_Q6_K_RDNA2 128+#define NWARPS_Q6_K_RDNA2 8+#define MMQ_X_Q6_K_RDNA1 32+#define MMQ_Y_Q6_K_RDNA1 64+#define NWARPS_Q6_K_RDNA1 8+#if defined(CUDA_USE_TENSOR_CORES)+#define MMQ_X_Q6_K_AMPERE 4+#define MMQ_Y_Q6_K_AMPERE 32+#define NWARPS_Q6_K_AMPERE 4+#else+#define MMQ_X_Q6_K_AMPERE 64+#define MMQ_Y_Q6_K_AMPERE 64+#define NWARPS_Q6_K_AMPERE 4+#endif+#define MMQ_X_Q6_K_PASCAL 64+#define MMQ_Y_Q6_K_PASCAL 64+#define NWARPS_Q6_K_PASCAL 8- size_t nb10 = cnb1[0];- size_t nb11 = cnb1[1];- size_t nb12 = cnb1[2];- size_t nb13 = cnb1[3];+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_RDNA2, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#elif __CUDA_ARCH__ < CC_VOLTA+ __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_PASCAL, 2)+#endif // __CUDA_ARCH__ < CC_VOLTA+ mul_mat_q6_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- size_t s0 = nb0 / sizeof(dst_t);- size_t s1 = nb1 / sizeof(dst_t);- size_t s2 = nb2 / sizeof(dst_t);- size_t s3 = nb3 / sizeof(dst_t);+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ const int mmq_x = MMQ_X_Q6_K_RDNA2;+ const int mmq_y = MMQ_Y_Q6_K_RDNA2;+ const int nwarps = NWARPS_Q6_K_RDNA2;+#else+ const int mmq_x = MMQ_X_Q6_K_RDNA1;+ const int mmq_y = MMQ_Y_Q6_K_RDNA1;+ const int nwarps = NWARPS_Q6_K_RDNA1;+#endif // defined(RDNA3) || defined(RDNA2)- size_t s10 = nb10 / sizeof(src1_t);- size_t s11 = nb11 / sizeof(src1_t);- size_t s12 = nb12 / sizeof(src1_t);- size_t s13 = nb13 / sizeof(src1_t);+ mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,+ load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- GGML_ASSERT(s0 == 1);- GGML_ASSERT(s10 == 1);+#elif __CUDA_ARCH__ >= CC_VOLTA+ const int mmq_x = MMQ_X_Q6_K_AMPERE;+ const int mmq_y = MMQ_Y_Q6_K_AMPERE;+ const int nwarps = NWARPS_Q6_K_AMPERE;- const int block_size = 128;+ mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,+ load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- int64_t hne0 = std::max(ne0/2LL, 1LL);+#elif __CUDA_ARCH__ >= MIN_CC_DP4A+ const int mmq_x = MMQ_X_Q6_K_PASCAL;+ const int mmq_y = MMQ_Y_Q6_K_PASCAL;+ const int nwarps = NWARPS_Q6_K_PASCAL;- dim3 block_dims;- block_dims.x = std::min<unsigned int>(hne0, block_size);- block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);- block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);+ mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,+ load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(vec_dot_q6_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_VOLTA+}- dim3 block_nums(- (hne0 + block_dims.x - 1) / block_dims.x,- (ne1 + block_dims.y - 1) / block_dims.y,- (ne2*ne3 + block_dims.z - 1) / block_dims.z- );+template <int ncols_y, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+// tell the compiler to use as many registers as it wants, see nwarps definition below+__launch_bounds__((ncols_y <= 4 ? 4 : 2)*WARP_SIZE, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void mul_mat_vec_q(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int nrows_y, const int nrows_dst) {- if (block_nums.z > 65535) {- // this is the maximum number of blocks in z direction, fallback to 1D grid kernel- int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;- k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne0, ne1, ne2, ne3,- ne10, ne11, ne12, ne13,- /* s0, */ s1, s2, s3,- /* s10, */ s11, s12, s13);- } else {- k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne0, ne1, ne2, ne3,- ne10, ne11, ne12, ne13,- /* s0, */ s1, s2, s3,- /* s10, */ s11, s12, s13);- }- }- }-};+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && (defined(RDNA2) || defined(RDNA3))+ constexpr int nwarps = 1;+ constexpr int rows_per_cuda_block = 1;+#else+ constexpr int nwarps = ncols_y <= 4 ? 4 : 2;+ constexpr int rows_per_cuda_block = ncols_y == 1 ? 1 : 2;+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && !defined(RDNA2) && !defined(RDNA3)-static void acc_f32_cuda(const float * x, const float * y, float * dst, const int n_elements,- const int ne10, const int ne11, const int ne12,- const int nb1, const int nb2, const int offset, cudaStream_t stream) {- int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;- acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset);-}+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;+ const int row0 = rows_per_cuda_block*blockIdx.x;+ const int blocks_per_row_x = ncols_x / qk;+ const int blocks_per_col_y = nrows_y / QK8_1;+ constexpr int blocks_per_iter = vdr * nwarps*WARP_SIZE / qi;-static void gelu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_GELU_BLOCK_SIZE - 1) / CUDA_GELU_BLOCK_SIZE;- gelu_f32<<<num_blocks, CUDA_GELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+// partial sum for each thread+ float tmp[ncols_y][rows_per_cuda_block] = {0.0f};-static void silu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_SILU_BLOCK_SIZE - 1) / CUDA_SILU_BLOCK_SIZE;- silu_f32<<<num_blocks, CUDA_SILU_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+ const block_q_t * x = (const block_q_t *) vx;+ const block_q8_1 * y = (const block_q8_1 *) vy;-static void gelu_quick_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_GELU_BLOCK_SIZE - 1) / CUDA_GELU_BLOCK_SIZE;- gelu_quick_f32<<<num_blocks, CUDA_GELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+ for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {+ const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx-static void tanh_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_TANH_BLOCK_SIZE - 1) / CUDA_TANH_BLOCK_SIZE;- tanh_f32<<<num_blocks, CUDA_TANH_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+ // x block quant index when casting the quants to int+ const int kqs = vdr * (tid % (qi/vdr));-static void relu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;- relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+#pragma unroll+ for (int j = 0; j < ncols_y; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+ tmp[j][i] += vec_dot_q_cuda(+ &x[kbx + (row0 + i)*blocks_per_row_x], &y[j*blocks_per_col_y + kby], kqs);+ }+ }+ }-static void hardsigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_HARDSIGMOID_BLOCK_SIZE - 1) / CUDA_HARDSIGMOID_BLOCK_SIZE;- hardsigmoid_f32<<<num_blocks, CUDA_HARDSIGMOID_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+ __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_y][rows_per_cuda_block][WARP_SIZE];+ if (threadIdx.y > 0) {+#pragma unroll+ for (int j = 0; j < ncols_y; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+ tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];+ }+ }+ }+ __syncthreads();+ if (threadIdx.y > 0) {+ return;+ }-static void hardswish_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_HARDSWISH_BLOCK_SIZE - 1) / CUDA_HARDSWISH_BLOCK_SIZE;- hardswish_f32<<<num_blocks, CUDA_HARDSWISH_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+ // sum up partial sums and write back result+#pragma unroll+ for (int j = 0; j < ncols_y; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+#pragma unroll+ for (int l = 0; l < nwarps-1; ++l) {+ tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];+ }+ tmp[j][i] = warp_reduce_sum(tmp[j][i]);+ }-static void leaky_relu_f32_cuda(const float * x, float * dst, const int k, const float negative_slope, cudaStream_t stream) {- const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;- leaky_relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k, negative_slope);+ if (threadIdx.x < rows_per_cuda_block) {+ dst[j*nrows_dst + row0 + threadIdx.x] = tmp[j][threadIdx.x];+ }+ }}-static void sqr_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_SQR_BLOCK_SIZE - 1) / CUDA_SQR_BLOCK_SIZE;- sqr_f32<<<num_blocks, CUDA_SQR_BLOCK_SIZE, 0, stream>>>(x, dst, k);-}+template <int qk, int qr, dequantize_kernel_t dequantize_kernel>+static __global__ void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows) {+ // qk = quantized weights per x block+ // qr = number of quantized weights per data value in x block+ const int row = blockIdx.x*blockDim.y + threadIdx.y;-static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {- GGML_ASSERT(ncols % WARP_SIZE == 0);- if (ncols < 1024) {- const dim3 block_dims(WARP_SIZE, 1, 1);- norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);- } else {- const dim3 block_dims(1024, 1, 1);- norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ if (row >= nrows) {+ return;}-}-static void group_norm_f32_cuda(const float * x, float * dst, const int num_groups, const int group_size, const int ne_elements, cudaStream_t stream) {- static const float eps = 1e-6f;- if (group_size < 1024) {- const dim3 block_dims(WARP_SIZE, 1, 1);- group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);- } else {- const dim3 block_dims(1024, 1, 1);- group_norm_f32<1024><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);+ const int tid = threadIdx.x;++ const int iter_stride = 2*GGML_CUDA_DMMV_X;+ const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter+ const int y_offset = qr == 1 ? 1 : qk/2;++// partial sum for each thread+#ifdef GGML_CUDA_F16+ half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics+#else+ float tmp = 0.0f;+#endif // GGML_CUDA_F16++ for (int i = 0; i < ncols; i += iter_stride) {+ const int col = i + vals_per_iter*tid;+ const int ib = (row*ncols + col)/qk; // x block index+ const int iqs = (col%qk)/qr; // x quant index+ const int iybs = col - col%qk; // y block start index++// processing >2 values per i iter is faster for fast GPUs+#pragma unroll+ for (int j = 0; j < vals_per_iter; j += 2) {+ // process 2 vals per j iter++ // dequantize+ // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val+ dfloat2 v;+ dequantize_kernel(vx, ib, iqs + j/qr, v);++ // matrix multiplication+ // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2+#ifdef GGML_CUDA_F16+ tmp += __hmul2(v, {+ y[iybs + iqs + j/qr + 0],+ y[iybs + iqs + j/qr + y_offset]+ });+#else+ tmp += v.x * y[iybs + iqs + j/qr + 0];+ tmp += v.y * y[iybs + iqs + j/qr + y_offset];+#endif // GGML_CUDA_F16+ }}-}-static void concat_f32_cuda(const float * x, const float * y, float * dst, const int ne0, int ne1, int ne2, int ne02, cudaStream_t stream) {- int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;- dim3 gridDim(num_blocks, ne1, ne2);- concat_f32<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);-}+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);-static void upscale_f32_cuda(const float * x, float * dst, const int ne00, const int ne01, const int ne02, const int scale_factor, cudaStream_t stream) {- int ne0 = (ne00 * scale_factor);- int num_blocks = (ne0 + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;- dim3 gridDim(num_blocks, (ne01 * scale_factor), ne02);- upscale_f32<<<gridDim, CUDA_UPSCALE_BLOCK_SIZE, 0, stream>>>(x, dst, ne00, ne00 * ne01, scale_factor);+ if (tid == 0) {+#ifdef GGML_CUDA_F16+ dst[row] = tmp.x + tmp.y;+#else+ dst[row] = tmp;+#endif // GGML_CUDA_F16+ }}-static void pad_f32_cuda(const float * x, float * dst,- const int ne00, const int ne01, const int ne02,- const int ne0, const int ne1, const int ne2, cudaStream_t stream) {- int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE;- dim3 gridDim(num_blocks, ne1, ne2);- pad_f32<<<gridDim, CUDA_PAD_BLOCK_SIZE, 0, stream>>>(x, dst, ne0, ne00, ne01, ne02);-}+static __global__ void mul_mat_p021_f16_f32(+ const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int nchannels_x, const int nchannels_y) {-static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {- GGML_ASSERT(ncols % WARP_SIZE == 0);- if (ncols < 1024) {- const dim3 block_dims(WARP_SIZE, 1, 1);- rms_norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);- } else {- const dim3 block_dims(1024, 1, 1);- rms_norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ const half * x = (const half *) vx;++ const int row_x = blockDim.y*blockIdx.y + threadIdx.y;+ const int channel = blockDim.z*blockIdx.z + threadIdx.z;+ const int channel_x = channel / (nchannels_y / nchannels_x);++ const int nrows_y = ncols_x;+ const int nrows_dst = nrows_x;+ const int row_dst = row_x;++ float tmp = 0.0f;++ for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {+ const int col_x = col_x0 + threadIdx.x;++ if (col_x >= ncols_x) {+ break;+ }++ // x is transposed and permuted+ const int ix = row_x*nchannels_x*ncols_x + channel_x*ncols_x + col_x;+ const float xi = __half2float(x[ix]);++ const int row_y = col_x;++ // y is not transposed but permuted+ const int iy = channel*nrows_y + row_y;++ tmp += xi * y[iy];}-}-static void quantize_row_q8_1_cuda(const float * x, void * vy, const int kx, const int ky, const int kx_padded, cudaStream_t stream) {- const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;- const dim3 num_blocks(block_num_x, ky, 1);- const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);- quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);-}+ // dst is not transposed and not permuted+ const int idst = channel*nrows_dst + row_dst;-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {- const int num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);- dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);-static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;- if (k % CUDA_Q8_0_NE_ALIGN == 0) {- const bool need_check = false;- dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);- } else {- const bool need_check = true;- dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ if (threadIdx.x == 0) {+ dst[idst] = tmp;}}-template<typename dst_t>-static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q2_K<<<nb, 32, 0, stream>>>(vx, y);-#endif-}+static __global__ void mul_mat_vec_nc_f16_f32( // nc == non-contiguous+ const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst, const int ncols_x, const int nrows_x,+ const int row_stride_x, const int channel_stride_x, const int channel_x_divisor) {-template<typename dst_t>-static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q3_K<<<nb, 32, 0, stream>>>(vx, y);-#endif-}+ const half * x = (const half *) vx;-template<typename dst_t>-static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb32 = k / 32;- const int nb = (k + 255) / 256;- dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);-}+ const int row_x = blockDim.y*blockIdx.y + threadIdx.y;+ const int channel = blockDim.z*blockIdx.z + threadIdx.z;+ const int channel_x = channel / channel_x_divisor;-template<typename dst_t>-static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb32 = k / 32;- const int nb = (k + 255) / 256;- dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);-}+ const int nrows_y = ncols_x;+ const int nrows_dst = nrows_x;+ const int row_dst = row_x;-template<typename dst_t>-static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);-}+ const int idst = channel*nrows_dst + row_dst;-template<typename dst_t>-static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q5_K<<<nb, 32, 0, stream>>>(vx, y);-#endif-}+ float tmp = 0.0f;-template<typename dst_t>-static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q6_K<<<nb, 32, 0, stream>>>(vx, y);-#endif-}+ for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {+ const int col_x = col_x0 + threadIdx.x;-template<typename dst_t>-static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);-}+ if (col_x >= ncols_x) {+ break;+ }-template<typename dst_t>-static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);-}+ const int row_y = col_x;-template<typename dst_t>-static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);-}+ const int ix = channel_x*channel_stride_x + row_x*row_stride_x + col_x;+ const int iy = channel*nrows_y + row_y;-template<typename dst_t>-static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);-}+ const float xi = __half2float(x[ix]);-template<typename dst_t>-static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);-}+ tmp += xi * y[iy];+ }-template<typename dst_t>-static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);-}+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);-template<typename dst_t>-static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = (k + QK_K - 1) / QK_K;- dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);+ if (threadIdx.x == 0) {+ dst[idst] = tmp;+ }}-template<typename dst_t>-static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {- const int nb = (k + QK_K - 1) / QK_K;- dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);-}+static __device__ void cpy_1_f32_f32(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ float * dsti = (float *) cdsti;-template <typename src_t, typename dst_t>-static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}--static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {- int id;- switch (type) {- case GGML_TYPE_Q4_0:- return dequantize_row_q4_0_cuda;- case GGML_TYPE_Q4_1:- return dequantize_row_q4_1_cuda;- case GGML_TYPE_Q5_0:- return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;- case GGML_TYPE_Q5_1:- return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;- case GGML_TYPE_Q8_0:- CUDA_CHECK(cudaGetDevice(&id));- if (g_device_caps[id].cc >= CC_PASCAL) {- return dequantize_block_q8_0_f16_cuda;- }- return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;- case GGML_TYPE_Q2_K:- return dequantize_row_q2_K_cuda;- case GGML_TYPE_Q3_K:- return dequantize_row_q3_K_cuda;- case GGML_TYPE_Q4_K:- return dequantize_row_q4_K_cuda;- case GGML_TYPE_Q5_K:- return dequantize_row_q5_K_cuda;- case GGML_TYPE_Q6_K:- return dequantize_row_q6_K_cuda;- case GGML_TYPE_IQ2_XXS:- return dequantize_row_iq2_xxs_cuda;- case GGML_TYPE_IQ2_XS:- return dequantize_row_iq2_xs_cuda;- case GGML_TYPE_IQ2_S:- return dequantize_row_iq2_s_cuda;- case GGML_TYPE_IQ3_XXS:- return dequantize_row_iq3_xxs_cuda;- case GGML_TYPE_IQ1_S:- return dequantize_row_iq1_s_cuda;- case GGML_TYPE_IQ4_NL:- return dequantize_row_iq4_nl_cuda;- case GGML_TYPE_IQ4_XS:- return dequantize_row_iq4_xs_cuda;- case GGML_TYPE_IQ3_S:- return dequantize_row_iq3_s_cuda;- case GGML_TYPE_F32:- return convert_unary_cuda<float>;- default:- return nullptr;- }+ *dsti = *xi;}-static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {- switch (type) {- case GGML_TYPE_Q4_0:- return dequantize_row_q4_0_cuda;- case GGML_TYPE_Q4_1:- return dequantize_row_q4_1_cuda;- case GGML_TYPE_Q5_0:- return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;- case GGML_TYPE_Q5_1:- return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;- case GGML_TYPE_Q8_0:- return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;- case GGML_TYPE_Q2_K:- return dequantize_row_q2_K_cuda;- case GGML_TYPE_Q3_K:- return dequantize_row_q3_K_cuda;- case GGML_TYPE_Q4_K:- return dequantize_row_q4_K_cuda;- case GGML_TYPE_Q5_K:- return dequantize_row_q5_K_cuda;- case GGML_TYPE_Q6_K:- return dequantize_row_q6_K_cuda;- case GGML_TYPE_IQ2_XXS:- return dequantize_row_iq2_xxs_cuda;- case GGML_TYPE_IQ2_XS:- return dequantize_row_iq2_xs_cuda;- case GGML_TYPE_IQ2_S:- return dequantize_row_iq2_s_cuda;- case GGML_TYPE_IQ3_XXS:- return dequantize_row_iq3_xxs_cuda;- case GGML_TYPE_IQ1_S:- return dequantize_row_iq1_s_cuda;- case GGML_TYPE_IQ4_NL:- return dequantize_row_iq4_nl_cuda;- case GGML_TYPE_IQ4_XS:- return dequantize_row_iq4_xs_cuda;- case GGML_TYPE_IQ3_S:- return dequantize_row_iq3_s_cuda;- case GGML_TYPE_F16:- return convert_unary_cuda<half>;- default:- return nullptr;- }-}+static __device__ void cpy_1_f32_f16(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ half * dsti = (half *) cdsti;-static void dequantize_mul_mat_vec_q4_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+ *dsti = __float2half(*xi);}-static void dequantize_mul_mat_vec_q4_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+static __device__ void cpy_1_f16_f16(const char * cxi, char * cdsti) {+ const half * xi = (const half *) cxi;+ half * dsti = (half *) cdsti;-static void dequantize_mul_mat_vec_q5_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+ *dsti = *xi;}-static void dequantize_mul_mat_vec_q5_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+static __device__ void cpy_1_f16_f32(const char * cxi, char * cdsti) {+ const half * xi = (const half *) cxi;+ float * dsti = (float *) cdsti;-static void dequantize_mul_mat_vec_q8_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+ *dsti = *xi;}-static void dequantize_mul_mat_vec_q2_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q2_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+template <cpy_kernel_t cpy_1>+static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,+ const int nb12, const int nb13) {+ const int64_t i = blockDim.x*blockIdx.x + threadIdx.x;-static void dequantize_mul_mat_vec_q3_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2 / K_QUANTS_PER_ITERATION;- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q3_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+ if (i >= ne) {+ return;+ }-static void dequantize_mul_mat_vec_q4_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2 / K_QUANTS_PER_ITERATION;- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q4_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+ // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor+ // then combine those indices with the corresponding byte offsets to get the total offsets+ const int64_t i03 = i/(ne00 * ne01 * ne02);+ const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);+ const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;+ const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;+ const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;++ const int64_t i13 = i/(ne10 * ne11 * ne12);+ const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);+ const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;+ const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;+ const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;-static void dequantize_mul_mat_vec_q5_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const dim3 block_dims(32, 1, 1);- dequantize_mul_mat_vec_q5_k<<<nrows, block_dims, 0, stream>>>(vx, y, dst, ncols);+ cpy_1(cx + x_offset, cdst + dst_offset);}-static void dequantize_mul_mat_vec_q6_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2 / K_QUANTS_PER_ITERATION;- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q6_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q8_0 * dsti = (block_q8_0 *) cdsti;-static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<1, 1, convert_f16>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+ float amax = 0.0f; // absolute max-template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot>-static void mul_mat_vec_q_cuda(- const void * vx, const void * vy, float * dst,- const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {+ for (int j = 0; j < QK8_0; j++) {+ const float v = xi[j];+ amax = fmaxf(amax, fabsf(v));+ }- GGML_ASSERT(ncols_x % qk == 0);- GGML_ASSERT(ncols_y <= MMVQ_MAX_BATCH_SIZE);+ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;- int id;- CUDA_CHECK(cudaGetDevice(&id));+ dsti->d = d;- int64_t nwarps = 1;- int64_t rows_per_cuda_block = 1;+ for (int j = 0; j < QK8_0; ++j) {+ const float x0 = xi[j]*id;- if (g_device_caps[id].cc < CC_RDNA2) { // NVIDIA and AMD older than RDNA2- switch(ncols_y) {- case 1:- nwarps = 4;- rows_per_cuda_block = 1;- break;- case 2:- case 3:- case 4:- nwarps = 4;- rows_per_cuda_block = 2;- break;- case 5:- case 6:- case 7:- case 8:- nwarps = 2;- rows_per_cuda_block = 2;- break;- default:- GGML_ASSERT(false);- break;- }+ dsti->qs[j] = roundf(x0);}- const int64_t nblocks = (nrows_x + rows_per_cuda_block - 1) / rows_per_cuda_block;- const dim3 block_nums(nblocks, 1, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+}- switch (ncols_y) {- case 1:- mul_mat_vec_q<1, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- case 2:- mul_mat_vec_q<2, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- case 3:- mul_mat_vec_q<3, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- case 4:- mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- case 5:- mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- case 6:- mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- case 7:- mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- case 8:- mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);- break;- default:- GGML_ASSERT(false);- break;+static __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q4_0 * dsti = (block_q4_0 *) cdsti;++ float amax = 0.0f;+ float vmax = 0.0f;++ for (int j = 0; j < QK4_0; ++j) {+ const float v = xi[j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ vmax = v;+ }}-}-static void ggml_mul_mat_q4_0_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+ const float d = vmax / -8;+ const float id = d ? 1.0f/d : 0.0f;- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+ dsti->d = d;- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q4_0_RDNA2;- mmq_y = MMQ_Y_Q4_0_RDNA2;- nwarps = NWARPS_Q4_0_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q4_0_RDNA1;- mmq_y = MMQ_Y_Q4_0_RDNA1;- nwarps = NWARPS_Q4_0_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q4_0_AMPERE;- mmq_y = MMQ_Y_Q4_0_AMPERE;- nwarps = NWARPS_Q4_0_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q4_0_PASCAL;- mmq_y = MMQ_Y_Q4_0_PASCAL;- nwarps = NWARPS_Q4_0_PASCAL;- } else {- GGML_ASSERT(false);- }+ for (int j = 0; j < QK4_0/2; ++j) {+ const float x0 = xi[0 + j]*id;+ const float x1 = xi[QK4_0/2 + j]*id;- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));+ const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ dsti->qs[j] = xi0;+ dsti->qs[j] |= xi1 << 4;}}-static void ggml_mul_mat_q4_1_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q4_1 * dsti = (block_q4_1 *) cdsti;- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+ float vmin = FLT_MAX;+ float vmax = -FLT_MAX;- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q4_1_RDNA2;- mmq_y = MMQ_Y_Q4_1_RDNA2;- nwarps = NWARPS_Q4_1_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q4_1_RDNA1;- mmq_y = MMQ_Y_Q4_1_RDNA1;- nwarps = NWARPS_Q4_1_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q4_1_AMPERE;- mmq_y = MMQ_Y_Q4_1_AMPERE;- nwarps = NWARPS_Q4_1_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q4_1_PASCAL;- mmq_y = MMQ_Y_Q4_1_PASCAL;- nwarps = NWARPS_Q4_1_PASCAL;- } else {- GGML_ASSERT(false);+ for (int j = 0; j < QK4_1; ++j) {+ const float v = xi[j];++ if (v < vmin) vmin = v;+ if (v > vmax) vmax = v;}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ const float d = (vmax - vmin) / ((1 << 4) - 1);+ const float id = d ? 1.0f/d : 0.0f;- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ dsti->dm.x = d;+ dsti->dm.y = vmin;++ for (int j = 0; j < QK4_1/2; ++j) {+ const float x0 = (xi[0 + j] - vmin)*id;+ const float x1 = (xi[QK4_1/2 + j] - vmin)*id;++ const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));+ const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));++ dsti->qs[j] = xi0;+ dsti->qs[j] |= xi1 << 4;}}-static void ggml_mul_mat_q5_0_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+static __device__ void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q5_0 * dsti = (block_q5_0 *) cdsti;- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+ float amax = 0.0f;+ float vmax = 0.0f;- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q5_0_RDNA2;- mmq_y = MMQ_Y_Q5_0_RDNA2;- nwarps = NWARPS_Q5_0_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q5_0_RDNA1;- mmq_y = MMQ_Y_Q5_0_RDNA1;- nwarps = NWARPS_Q5_0_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q5_0_AMPERE;- mmq_y = MMQ_Y_Q5_0_AMPERE;- nwarps = NWARPS_Q5_0_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q5_0_PASCAL;- mmq_y = MMQ_Y_Q5_0_PASCAL;- nwarps = NWARPS_Q5_0_PASCAL;- } else {- GGML_ASSERT(false);+ for (int j = 0; j < QK5_0; ++j) {+ const float v = xi[j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ vmax = v;+ }}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ const float d = vmax / -16;+ const float id = d ? 1.0f/d : 0.0f;- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ dsti->d = d;++ uint32_t qh = 0;+ for (int j = 0; j < QK5_0/2; ++j) {+ const float x0 = xi[0 + j]*id;+ const float x1 = xi[QK5_0/2 + j]*id;++ const uint8_t xi0 = min(31, (int8_t)(x0 + 16.5f));+ const uint8_t xi1 = min(31, (int8_t)(x1 + 16.5f));++ dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);}+ memcpy(dsti->qh, &qh, sizeof(qh));}-static void ggml_mul_mat_q5_1_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+static __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q5_1 * dsti = (block_q5_1 *) cdsti;- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+ float min = xi[0];+ float max = xi[0];- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q5_1_RDNA2;- mmq_y = MMQ_Y_Q5_1_RDNA2;- nwarps = NWARPS_Q5_1_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q5_1_RDNA1;- mmq_y = MMQ_Y_Q5_1_RDNA1;- nwarps = NWARPS_Q5_1_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q5_1_AMPERE;- mmq_y = MMQ_Y_Q5_1_AMPERE;- nwarps = NWARPS_Q5_1_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q5_1_PASCAL;- mmq_y = MMQ_Y_Q5_1_PASCAL;- nwarps = NWARPS_Q5_1_PASCAL;- } else {- GGML_ASSERT(false);+ for (int j = 1; j < QK5_1; ++j) {+ const float v = xi[j];+ min = v < min ? v : min;+ max = v > max ? v : max;}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ const float d = (max - min) / 31;+ const float id = d ? 1.0f/d : 0.0f;- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ dsti->dm.x = d;+ dsti->dm.y = min;++ uint32_t qh = 0;+ for (int j = 0; j < QK5_1/2; ++j) {+ const float x0 = (xi[0 + j] - min)*id;+ const float x1 = (xi[QK5_1/2 + j] - min)*id;++ const uint8_t xi0 = (uint8_t)(x0 + 0.5f);+ const uint8_t xi1 = (uint8_t)(x1 + 0.5f);++ dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);}+ memcpy(dsti->qh, &qh, sizeof(qh));}-static void ggml_mul_mat_q8_0_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+static __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) {+ if (x <= val[0]) return 0;+ if (x >= val[n-1]) return n-1;+ int ml = 0, mu = n-1;+ while (mu-ml > 1) {+ int mav = (ml+mu)/2;+ if (x < val[mav]) mu = mav; else ml = mav;+ }+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;+}- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+static __device__ void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_iq4_nl * dsti = (block_iq4_nl *) cdsti;- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q8_0_RDNA2;- mmq_y = MMQ_Y_Q8_0_RDNA2;- nwarps = NWARPS_Q8_0_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q8_0_RDNA1;- mmq_y = MMQ_Y_Q8_0_RDNA1;- nwarps = NWARPS_Q8_0_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q8_0_AMPERE;- mmq_y = MMQ_Y_Q8_0_AMPERE;- nwarps = NWARPS_Q8_0_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q8_0_PASCAL;- mmq_y = MMQ_Y_Q8_0_PASCAL;- nwarps = NWARPS_Q8_0_PASCAL;- } else {- GGML_ASSERT(false);+ float amax = 0.0f;+ float vmax = 0.0f;++ for (int j = 0; j < QK4_NL; ++j) {+ const float v = xi[j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ vmax = v;+ }}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ float d = vmax / kvalues_iq4nl[0];+ const float id = d ? 1.0f/d : 0.0f;- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ float sumqx = 0, sumq2 = 0;+ for (int j = 0; j < QK4_NL/2; ++j) {+ const float x0 = xi[0 + j]*id;+ const float x1 = xi[QK4_NL/2 + j]*id;+ const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0);+ const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1);+ dsti->qs[j] = xi0 | (xi1 << 4);+ const float v0 = kvalues_iq4nl[xi0];+ const float v1 = kvalues_iq4nl[xi1];+ const float w0 = xi[0 + j]*xi[0 + j];+ const float w1 = xi[QK4_NL/2 + j]*xi[QK4_NL/2 + j];+ sumqx += w0*v0*xi[j] + w1*v1*xi[QK4_NL/2 + j];+ sumq2 += w0*v0*v0 + w1*v1*v1;}++ dsti->d = sumq2 > 0 ? sumqx/sumq2 : d;}-static void ggml_mul_mat_q2_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+template <cpy_kernel_t cpy_blck, int qk>+static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,+ const int nb12, const int nb13) {+ const int i = (blockDim.x*blockIdx.x + threadIdx.x)*qk;- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q2_K_RDNA2;- mmq_y = MMQ_Y_Q2_K_RDNA2;- nwarps = NWARPS_Q2_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q2_K_RDNA1;- mmq_y = MMQ_Y_Q2_K_RDNA1;- nwarps = NWARPS_Q2_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q2_K_AMPERE;- mmq_y = MMQ_Y_Q2_K_AMPERE;- nwarps = NWARPS_Q2_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q2_K_PASCAL;- mmq_y = MMQ_Y_Q2_K_PASCAL;- nwarps = NWARPS_Q2_K_PASCAL;- } else {- GGML_ASSERT(false);+ if (i >= ne) {+ return;}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ const int i03 = i/(ne00 * ne01 * ne02);+ const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);+ const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;+ const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;+ const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }+ const int i13 = i/(ne10 * ne11 * ne12);+ const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);+ const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;+ const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;+ const int dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;++ cpy_blck(cx + x_offset, cdst + dst_offset);}-static void ggml_mul_mat_q3_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+static __device__ float rope_yarn_ramp(const float low, const float high, const int i0) {+ const float y = (i0 / 2 - low) / max(0.001f, high - low);+ return 1.0f - min(1.0f, max(0.0f, y));+}-#if QK_K == 256+struct rope_corr_dims {+ float v[4];+};- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn+// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.+static __device__ void rope_yarn(+ float theta_extrap, float freq_scale, rope_corr_dims corr_dims, int64_t i0, float ext_factor, float mscale,+ float * cos_theta, float * sin_theta+) {+ // Get n-d rotational scaling corrected for extrapolation+ float theta_interp = freq_scale * theta_extrap;+ float theta = theta_interp;+ if (ext_factor != 0.0f) {+ float ramp_mix = rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q3_K_RDNA2;- mmq_y = MMQ_Y_Q3_K_RDNA2;- nwarps = NWARPS_Q3_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q3_K_RDNA1;- mmq_y = MMQ_Y_Q3_K_RDNA1;- nwarps = NWARPS_Q3_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q3_K_AMPERE;- mmq_y = MMQ_Y_Q3_K_AMPERE;- nwarps = NWARPS_Q3_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q3_K_PASCAL;- mmq_y = MMQ_Y_Q3_K_PASCAL;- nwarps = NWARPS_Q3_K_PASCAL;- } else {- GGML_ASSERT(false);+ // Get n-d magnitude scaling corrected for interpolation+ mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);}+ *cos_theta = cosf(theta) * mscale;+ *sin_theta = sinf(theta) * mscale;+}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+// rope == RoPE == rotary positional embedding+template<typename T, bool has_pos>+static __global__ void rope(+ const T * x, T * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,+ float ext_factor, float attn_factor, rope_corr_dims corr_dims+) {+ const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ if (col >= ncols) {+ return;}-#endif-}-static void ggml_mul_mat_q4_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+ const int row = blockDim.x*blockIdx.x + threadIdx.x;+ const int i = row*ncols + col;+ const int i2 = row/p_delta_rows;- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+ const int p = has_pos ? pos[i2] : 0;+ const float theta_base = p*powf(freq_base, -float(col)/ncols);- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q4_K_RDNA2;- mmq_y = MMQ_Y_Q4_K_RDNA2;- nwarps = NWARPS_Q4_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q4_K_RDNA1;- mmq_y = MMQ_Y_Q4_K_RDNA1;- nwarps = NWARPS_Q4_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q4_K_AMPERE;- mmq_y = MMQ_Y_Q4_K_AMPERE;- nwarps = NWARPS_Q4_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q4_K_PASCAL;- mmq_y = MMQ_Y_Q4_K_PASCAL;- nwarps = NWARPS_Q4_K_PASCAL;- } else {- GGML_ASSERT(false);- }+ float cos_theta, sin_theta;+ rope_yarn(theta_base, freq_scale, corr_dims, col, ext_factor, attn_factor, &cos_theta, &sin_theta);- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ const float x0 = x[i + 0];+ const float x1 = x[i + 1];- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }+ dst[i + 0] = x0*cos_theta - x1*sin_theta;+ dst[i + 1] = x0*sin_theta + x1*cos_theta;}-static void ggml_mul_mat_q5_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+template<typename T, bool has_pos>+static __global__ void rope_neox(+ const T * x, T * dst, int ncols, int n_dims, const int32_t * pos, float freq_scale, int p_delta_rows,+ float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, float inv_ndims+) {+ const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q5_K_RDNA2;- mmq_y = MMQ_Y_Q5_K_RDNA2;- nwarps = NWARPS_Q5_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q5_K_RDNA1;- mmq_y = MMQ_Y_Q5_K_RDNA1;- nwarps = NWARPS_Q5_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q5_K_AMPERE;- mmq_y = MMQ_Y_Q5_K_AMPERE;- nwarps = NWARPS_Q5_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q5_K_PASCAL;- mmq_y = MMQ_Y_Q5_K_PASCAL;- nwarps = NWARPS_Q5_K_PASCAL;- } else {- GGML_ASSERT(false);+ if (col >= ncols) {+ return;}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}+ const int row = blockDim.x*blockIdx.x + threadIdx.x;+ const int ib = col / n_dims;+ const int ic = col % n_dims;-static void ggml_mul_mat_q6_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {+ if (ib > 0) {+ const int i = row*ncols + ib*n_dims + ic;- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_device_caps[id].cc;+ dst[i + 0] = x[i + 0];+ dst[i + 1] = x[i + 1];- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q6_K_RDNA2;- mmq_y = MMQ_Y_Q6_K_RDNA2;- nwarps = NWARPS_Q6_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q6_K_RDNA1;- mmq_y = MMQ_Y_Q6_K_RDNA1;- nwarps = NWARPS_Q6_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q6_K_AMPERE;- mmq_y = MMQ_Y_Q6_K_AMPERE;- nwarps = NWARPS_Q6_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q6_K_PASCAL;- mmq_y = MMQ_Y_Q6_K_PASCAL;- nwarps = NWARPS_Q6_K_PASCAL;- } else {- GGML_ASSERT(false);+ return;}- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+ const int i = row*ncols + ib*n_dims + ic/2;+ const int i2 = row/p_delta_rows;- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}+ float cur_rot = inv_ndims * ic - ib;-static void ggml_mul_mat_p021_f16_f32_cuda(- const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x,- const int nchannels_x, const int nchannels_y, cudaStream_t stream) {+ const int p = has_pos ? pos[i2] : 0;+ const float theta_base = p*freq_scale*powf(theta_scale, col/2.0f);- const dim3 block_nums(1, nrows_x, nchannels_y);- const dim3 block_dims(WARP_SIZE, 1, 1);- mul_mat_p021_f16_f32<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols_x, nrows_x, nchannels_x, nchannels_y);-}+ float cos_theta, sin_theta;+ rope_yarn(theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);-static void ggml_mul_mat_vec_nc_f16_f32_cuda(- const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x, const int row_stride_x,- const int nchannels_x, const int nchannels_y, const int channel_stride_x, cudaStream_t stream) {+ const float x0 = x[i + 0];+ const float x1 = x[i + n_dims/2];- const dim3 block_nums(1, nrows_x, nchannels_y);- const dim3 block_dims(WARP_SIZE, 1, 1);- mul_mat_vec_nc_f16_f32<<<block_nums, block_dims, 0, stream>>>- (vx, y, dst, ncols_x, nrows_x, row_stride_x, channel_stride_x, nchannels_y/nchannels_x);+ dst[i + 0] = x0*cos_theta - x1*sin_theta;+ dst[i + n_dims/2] = x0*sin_theta + x1*cos_theta;}+static __global__ void rope_glm_f32(+ const float * x, float * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,+ int n_ctx+) {+ const int col = blockDim.x*blockIdx.x + threadIdx.x;+ const int half_n_dims = ncols/4;-static void ggml_cpy_f16_f32_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ if (col >= half_n_dims) {+ return;+ }- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f16_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ const int row = blockDim.y*blockIdx.y + threadIdx.y;+ const int i = row*ncols + col;+ const int i2 = row/p_delta_rows;-static void ggml_cpy_f32_f32_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ const float col_theta_scale = powf(freq_base, -2.0f*col/ncols);+ // FIXME: this is likely wrong+ const int p = pos != nullptr ? pos[i2] : 0;- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f32_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ const float theta = min(p, n_ctx - 2)*freq_scale*col_theta_scale;+ const float sin_theta = sinf(theta);+ const float cos_theta = cosf(theta);-static void ggml_cpy_f32_f16_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ const float x0 = x[i + 0];+ const float x1 = x[i + half_n_dims];- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f32_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ dst[i + 0] = x0*cos_theta - x1*sin_theta;+ dst[i + half_n_dims] = x0*sin_theta + x1*cos_theta;-static void ggml_cpy_f32_q8_0_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ const float block_theta = ((float)max(p - n_ctx - 2, 0))*col_theta_scale;+ const float sin_block_theta = sinf(block_theta);+ const float cos_block_theta = cosf(block_theta);- GGML_ASSERT(ne % QK8_0 == 0);- const int num_blocks = ne / QK8_0;- cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+ const float x2 = x[i + half_n_dims * 2];+ const float x3 = x[i + half_n_dims * 3];++ dst[i + half_n_dims * 2] = x2*cos_block_theta - x3*sin_block_theta;+ dst[i + half_n_dims * 3] = x2*sin_block_theta + x3*cos_block_theta;}-static void ggml_cpy_f32_q4_0_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+static __global__ void alibi_f32(const float * x, float * dst, const int ncols, const int k_rows,+ const int n_heads_log2_floor, const float m0, const float m1) {+ const int col = blockDim.x*blockIdx.x + threadIdx.x;- GGML_ASSERT(ne % QK4_0 == 0);- const int num_blocks = ne / QK4_0;- cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ if (col >= ncols) {+ return;+ }-static void ggml_cpy_f32_q4_1_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ const int row = blockDim.y*blockIdx.y + threadIdx.y;+ const int i = row*ncols + col;- GGML_ASSERT(ne % QK4_1 == 0);- const int num_blocks = ne / QK4_1;- cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ const int k = row/k_rows;-static void ggml_cpy_f16_f16_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ float m_k;+ if (k < n_heads_log2_floor) {+ m_k = powf(m0, k + 1);+ } else {+ m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);+ }- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f16_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+ dst[i] = col * m_k + x[i];}+static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {+ const int row = blockIdx.x;+ const int col = threadIdx.x;+ float sum = 0.0f;+ for (int i = col; i < ncols; i += blockDim.x) {+ sum += x[row * ncols + i];+ }-static void scale_f32_cuda(const float * x, float * dst, const float scale, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;- scale_f32<<<num_blocks, CUDA_SCALE_BLOCK_SIZE, 0, stream>>>(x, dst, scale, k);-}+ sum = warp_reduce_sum(sum);-static void clamp_f32_cuda(const float * x, float * dst, const float min, const float max, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE;- clamp_f32<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, k);+ if (col == 0) {+ dst[row] = sum;+ }}template<typename T>-static void rope_cuda(- const T * x, T * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,- float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream-) {- GGML_ASSERT(ncols % 2 == 0);- const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);- const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);- const dim3 block_nums(nrows, num_blocks_x, 1);- if (pos == nullptr) {- rope<T, false><<<block_nums, block_dims, 0, stream>>>(- x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims- );- } else {- rope<T, true><<<block_nums, block_dims, 0, stream>>>(- x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims- );- }+static inline __device__ void ggml_cuda_swap(T & a, T & b) {+ T tmp = a;+ a = b;+ b = tmp;}-template<typename T>-static void rope_neox_cuda(- const T * x, T * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,- float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream-) {- GGML_ASSERT(ncols % 2 == 0);- const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);- const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);- const dim3 block_nums(nrows, num_blocks_x, 1);+template<ggml_sort_order order>+static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols) {+ // bitonic sort+ int col = threadIdx.x;+ int row = blockIdx.y;- const float theta_scale = powf(freq_base, -2.0f/n_dims);- const float inv_ndims = -1.0f / n_dims;+ if (col >= ncols) return;- if (pos == nullptr) {- rope_neox<T, false><<<block_nums, block_dims, 0, stream>>>(- x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,- theta_scale, inv_ndims- );- } else {- rope_neox<T, true><<<block_nums, block_dims, 0, stream>>>(- x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,- theta_scale, inv_ndims- );- }-}+ const float * x_row = x + row * ncols;+ int * dst_row = dst + row * ncols;-static void rope_glm_f32_cuda(- const float * x, float * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,- float freq_base, int n_ctx, cudaStream_t stream-) {- GGML_ASSERT(ncols % 4 == 0);- const dim3 block_dims(CUDA_ROPE_BLOCK_SIZE/4, 1, 1);- const int num_blocks_x = (ncols + CUDA_ROPE_BLOCK_SIZE - 1) / CUDA_ROPE_BLOCK_SIZE;- const dim3 block_nums(num_blocks_x, nrows, 1);- rope_glm_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, n_ctx);-}--static void alibi_f32_cuda(const float * x, float * dst, const int ncols, const int nrows,- const int k_rows, const int n_heads_log2_floor, const float m0,- const float m1, cudaStream_t stream) {- const dim3 block_dims(CUDA_ALIBI_BLOCK_SIZE, 1, 1);- const int num_blocks_x = (ncols + CUDA_ALIBI_BLOCK_SIZE - 1) / (CUDA_ALIBI_BLOCK_SIZE);- const dim3 block_nums(num_blocks_x, nrows, 1);- alibi_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, k_rows, n_heads_log2_floor, m0, m1);-}+ // initialize indices+ if (col < ncols) {+ dst_row[col] = col;+ }+ __syncthreads();-static void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- const dim3 block_dims(WARP_SIZE, 1, 1);- const dim3 block_nums(nrows, 1, 1);- k_sum_rows_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);+ for (int k = 2; k <= ncols; k *= 2) {+ for (int j = k / 2; j > 0; j /= 2) {+ int ixj = col ^ j;+ if (ixj > col) {+ if ((col & k) == 0) {+ if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) {+ ggml_cuda_swap(dst_row[col], dst_row[ixj]);+ }+ } else {+ if (order == GGML_SORT_ORDER_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) {+ ggml_cuda_swap(dst_row[col], dst_row[ixj]);+ }+ }+ }+ __syncthreads();+ }+ }}-static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {- // bitonic sort requires ncols to be power of 2- GGML_ASSERT((ncols & (ncols - 1)) == 0);+static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {+ const int col = blockDim.y*blockIdx.y + threadIdx.y;+ const int row = blockDim.x*blockIdx.x + threadIdx.x;- const dim3 block_dims(ncols, 1, 1);- const dim3 block_nums(1, nrows, 1);- if (order == GGML_SORT_ORDER_ASC) {- k_argsort_f32_i32<GGML_SORT_ORDER_ASC><<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);- } else if (order == GGML_SORT_ORDER_DESC) {- k_argsort_f32_i32<GGML_SORT_ORDER_DESC><<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);- } else {- GGML_ASSERT(false);+ if (col >= ncols) {+ return;}-}-static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {- const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);- const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;- const dim3 block_nums(nrows_x, block_num_x, 1);- diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);+ const int i = row*ncols + col;+ //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];+ //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU+ dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;}-static void soft_max_f32_cuda(const float * x, const float * mask, const float * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {- int nth = WARP_SIZE;- while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;- const dim3 block_dims(nth, 1, 1);- const dim3 block_nums(nrows_x, 1, 1);- const size_t shmem = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE)*sizeof(float);- static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted.");+template <bool vals_smem, int ncols_template, int block_size_template>+static __global__ void soft_max_f32(const float * x, const float * mask, const float * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {+ const int ncols = ncols_template == 0 ? ncols_par : ncols_template;- const uint32_t n_head_kv = nrows_x/nrows_y;- const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv));+ const int tid = threadIdx.x;+ const int rowx = blockIdx.x;+ const int rowy = rowx % nrows_y; // broadcast the mask in the row dimension- const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);+ const int block_size = block_size_template == 0 ? blockDim.x : block_size_template;- if (shmem < g_device_caps[g_main_device].smpb) {- switch (ncols_x) {- case 32:- soft_max_f32<true, 32, 32><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- case 64:- soft_max_f32<true, 64, 64><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- case 128:- soft_max_f32<true, 128, 128><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- case 256:- soft_max_f32<true, 256, 256><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- case 512:- soft_max_f32<true, 512, 512><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- case 1024:- soft_max_f32<true, 1024, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- case 2048:- soft_max_f32<true, 2048, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- case 4096:- soft_max_f32<true, 4096, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- default:- soft_max_f32<true, 0, 0><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- break;- }- } else {- const size_t shmem_low = WARP_SIZE*sizeof(float);- soft_max_f32<false, 0, 0><<<block_nums, block_dims, shmem_low, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);- }-}+ const int warp_id = threadIdx.x / WARP_SIZE;+ const int lane_id = threadIdx.x % WARP_SIZE;-template <typename T>-static void im2col_cuda(const float* x, T* dst,- int IW, int IH, int OW, int OH, int KW, int KH, int IC,- int batch, int batch_offset, int offset_delta,- int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {- const int parallel_elements = OW * KW * KH;- const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;- dim3 block_nums(num_blocks, OH, batch * IC);- im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);-}+ float slope = 0.0f;-// buffer pool for cuda-#define MAX_CUDA_BUFFERS 256+ // ALiBi+ if (max_bias > 0.0f) {+ const int h = rowx/nrows_y; // head index-struct scoped_spin_lock {- std::atomic_flag& lock;- scoped_spin_lock(std::atomic_flag& lock) : lock(lock) {- while (lock.test_and_set(std::memory_order_acquire)) {- ; // spin- }- }- ~scoped_spin_lock() {- lock.clear(std::memory_order_release);+ const float base = h < n_head_log2 ? m0 : m1;+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ slope = powf(base, exp);}- scoped_spin_lock(const scoped_spin_lock&) = delete;- scoped_spin_lock& operator=(const scoped_spin_lock&) = delete;-};-static std::atomic_flag g_cuda_pool_lock = ATOMIC_FLAG_INIT;+ extern __shared__ float data_soft_max_f32[];+ float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication+ // shared memory buffer to cache values between iterations:+ float * vals = vals_smem ? buf_iw + WARP_SIZE : dst + rowx*ncols;-// #define DEBUG_CUDA_MALLOC-struct ggml_cuda_buffer {- void * ptr = nullptr;- size_t size = 0;-};+ float max_val = -INFINITY;-static ggml_cuda_buffer g_cuda_buffer_pool[GGML_CUDA_MAX_DEVICES][MAX_CUDA_BUFFERS];-static size_t g_cuda_pool_size[GGML_CUDA_MAX_DEVICES] = {0};+#pragma unroll+ for (int col0 = 0; col0 < ncols; col0 += block_size) {+ const int col = col0 + tid;-static void * ggml_cuda_pool_malloc_leg(int device, size_t size, size_t * actual_size) {- scoped_spin_lock lock(g_cuda_pool_lock);-#ifdef DEBUG_CUDA_MALLOC- int nnz = 0;- size_t max_size = 0;-#endif- size_t best_diff = 1ull << 36;- int ibest = -1;- for (int i = 0; i < MAX_CUDA_BUFFERS; ++i) {- ggml_cuda_buffer& b = g_cuda_buffer_pool[device][i];- if (b.ptr != nullptr) {-#ifdef DEBUG_CUDA_MALLOC- ++nnz;- if (b.size > max_size) max_size = b.size;-#endif- if (b.size >= size) {- size_t diff = b.size - size;- if (diff < best_diff) {- best_diff = diff;- ibest = i;- if (!best_diff) {- void * ptr = b.ptr;- *actual_size = b.size;- b.ptr = nullptr;- b.size = 0;- return ptr;- }- }- }+ if (ncols_template == 0 && col >= ncols) {+ break;}- }- if (ibest >= 0) {- ggml_cuda_buffer& b = g_cuda_buffer_pool[device][ibest];- void * ptr = b.ptr;- *actual_size = b.size;- b.ptr = nullptr;- b.size = 0;- return ptr;- }- void * ptr;- size_t look_ahead_size = (size_t) (1.05 * size);- look_ahead_size = 256 * ((look_ahead_size + 255)/256);- ggml_cuda_set_device(device);- CUDA_CHECK(cudaMalloc((void **) &ptr, look_ahead_size));- *actual_size = look_ahead_size;- g_cuda_pool_size[device] += look_ahead_size;-#ifdef DEBUG_CUDA_MALLOC- fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz,- (uint32_t)(max_size/1024/1024), (uint32_t)(g_cuda_pool_size[device]/1024/1024), (uint32_t)(size/1024/1024));-#endif- return ptr;-}-static void ggml_cuda_pool_free_leg(int device, void * ptr, size_t size) {- scoped_spin_lock lock(g_cuda_pool_lock);-- for (int i = 0; i < MAX_CUDA_BUFFERS; ++i) {- ggml_cuda_buffer& b = g_cuda_buffer_pool[device][i];- if (b.ptr == nullptr) {- b.ptr = ptr;- b.size = size;- return;- }- }- fprintf(stderr, "WARNING: cuda buffer pool full, increase MAX_CUDA_BUFFERS\n");- ggml_cuda_set_device(device);- CUDA_CHECK(cudaFree(ptr));- g_cuda_pool_size[device] -= size;-}+ const int ix = rowx*ncols + col;+ const int iy = rowy*ncols + col;-#if !defined(GGML_USE_HIPBLAS)-// pool with virtual memory-static CUdeviceptr g_cuda_pool_addr[GGML_CUDA_MAX_DEVICES] = {0};-static size_t g_cuda_pool_used[GGML_CUDA_MAX_DEVICES] = {0};-static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB+ const float val = x[ix]*scale + (mask ? mask[iy] : 0.0f) + (pos ? slope*pos[col] : 0.0f);-static void * ggml_cuda_pool_malloc_vmm(int device, size_t size, size_t * actual_size) {- scoped_spin_lock lock(g_cuda_pool_lock);+ vals[col] = val;+ max_val = max(max_val, val);+ }- // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types- const size_t alignment = 128;- size = alignment * ((size + alignment - 1) / alignment);+ // find the max value in the block+ max_val = warp_reduce_max(max_val);+ if (block_size > WARP_SIZE) {+ if (warp_id == 0) {+ buf_iw[lane_id] = -INFINITY;+ }+ __syncthreads();- size_t avail = g_cuda_pool_size[device] - g_cuda_pool_used[device];+ if (lane_id == 0) {+ buf_iw[warp_id] = max_val;+ }+ __syncthreads();- if (size > avail) {- // round up to the next multiple of the granularity- size_t reserve_size = size - avail;- const size_t granularity = g_device_caps[device].vmm_granularity;- reserve_size = granularity * ((reserve_size + granularity - 1) / granularity);+ max_val = buf_iw[lane_id];+ max_val = warp_reduce_max(max_val);+ }- GGML_ASSERT(g_cuda_pool_size[device] + reserve_size <= CUDA_POOL_VMM_MAX_SIZE);+ float tmp = 0.0f; // partial sum- // allocate more physical memory- CUmemAllocationProp prop = {};- prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;- prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;- prop.location.id = device;- CUmemGenericAllocationHandle handle;- CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0));+#pragma unroll+ for (int col0 = 0; col0 < ncols; col0 += block_size) {+ const int col = col0 + tid;- // reserve virtual address space (if not already reserved)- if (g_cuda_pool_addr[device] == 0) {- CU_CHECK(cuMemAddressReserve(&g_cuda_pool_addr[device], CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0));+ if (ncols_template == 0 && col >= ncols) {+ break;}- // map at the end of the pool- CU_CHECK(cuMemMap(g_cuda_pool_addr[device] + g_cuda_pool_size[device], reserve_size, 0, handle, 0));-- // the memory allocation handle is no longer needed after mapping- CU_CHECK(cuMemRelease(handle));+ const float val = expf(vals[col] - max_val);+ tmp += val;+ vals[col] = val;+ }- // set access- CUmemAccessDesc access = {};- access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;- access.location.id = device;- access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;- CU_CHECK(cuMemSetAccess(g_cuda_pool_addr[device] + g_cuda_pool_size[device], reserve_size, &access, 1));+ // find the sum of exps in the block+ tmp = warp_reduce_sum(tmp);+ if (block_size > WARP_SIZE) {+ __syncthreads();+ if (warp_id == 0) {+ buf_iw[lane_id] = 0.0f;+ }+ __syncthreads();- // add to the pool- g_cuda_pool_size[device] += reserve_size;+ if (lane_id == 0) {+ buf_iw[warp_id] = tmp;+ }+ __syncthreads();- //printf("cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\n",- // id, (unsigned long long) (g_cuda_pool_size[id]/1024/1024),- // (unsigned long long) (reserve_size/1024/1024));+ tmp = buf_iw[lane_id];+ tmp = warp_reduce_sum(tmp);}- GGML_ASSERT(g_cuda_pool_addr[device] != 0);+ const float inv_sum = 1.0f / tmp;- void * ptr = (void *) (g_cuda_pool_addr[device] + g_cuda_pool_used[device]);- *actual_size = size;- g_cuda_pool_used[device] += size;+#pragma unroll+ for (int col0 = 0; col0 < ncols; col0 += block_size) {+ const int col = col0 + tid;-#ifdef DEBUG_CUDA_MALLOC- printf("cuda pool[%d]: allocated %llu bytes at %llx\n", device, (unsigned long long) size, ptr);-#endif+ if (ncols_template == 0 && col >= ncols) {+ return;+ }- return ptr;+ const int idst = rowx*ncols + col;+ dst[idst] = vals[col] * inv_sum;+ }}-static void ggml_cuda_pool_free_vmm(int device, void * ptr, size_t size) {- scoped_spin_lock lock(g_cuda_pool_lock);--#ifdef DEBUG_CUDA_MALLOC- printf("cuda pool[%d]: freed %llu bytes at %llx\n", device, (unsigned long long) size, ptr);-#endif+static __global__ void scale_f32(const float * x, float * dst, const float scale, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;- g_cuda_pool_used[device] -= size;+ if (i >= k) {+ return;+ }- // all deallocations must be in reverse order of the allocations- GGML_ASSERT(ptr == (void *) (g_cuda_pool_addr[device] + g_cuda_pool_used[device]));+ dst[i] = scale * x[i];}-static void * ggml_cuda_pool_malloc(int device, size_t size, size_t * actual_size) {- if (g_device_caps[device].vmm) {- return ggml_cuda_pool_malloc_vmm(device, size, actual_size);- } else {- return ggml_cuda_pool_malloc_leg(device, size, actual_size);- }-}+static __global__ void clamp_f32(const float * x, float * dst, const float min, const float max, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;-static void ggml_cuda_pool_free(int device, void * ptr, size_t size) {- if (g_device_caps[device].vmm) {- ggml_cuda_pool_free_vmm(device, ptr, size);- } else {- ggml_cuda_pool_free_leg(device, ptr, size);+ if (i >= k) {+ return;}++ dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]);}-#else-#define ggml_cuda_pool_malloc ggml_cuda_pool_malloc_leg-#define ggml_cuda_pool_free ggml_cuda_pool_free_leg-#endif // !defined(GGML_USE_HIPBLAS)-template<typename T>-struct cuda_pool_alloc {- int device = -1;- T * ptr = nullptr;- size_t actual_size = 0;+template <typename T>+static __global__ void im2col_kernel(+ const float * x, T * dst, int64_t batch_offset,+ int64_t offset_delta, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW,+ int s0, int s1, int p0, int p1, int d0, int d1) {+ const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;+ if (i >= pelements) {+ return;+ }- // size is in number of elements- T * alloc(size_t size) {- GGML_ASSERT(ptr == nullptr);- CUDA_CHECK(cudaGetDevice(&device));- ptr = (T *) ggml_cuda_pool_malloc(device, size * sizeof(T), &this->actual_size);- return ptr;+ const int64_t ksize = OW * (KH > 1 ? KW : 1);+ const int64_t kx = i / ksize;+ const int64_t kd = kx * ksize;+ const int64_t ky = (i - kd) / OW;+ const int64_t ix = i % OW;++ const int64_t oh = blockIdx.y;+ const int64_t batch = blockIdx.z / IC;+ const int64_t ic = blockIdx.z % IC;++ const int64_t iiw = ix * s0 + kx * d0 - p0;+ const int64_t iih = oh * s1 + ky * d1 - p1;++ const int64_t offset_dst =+ ((batch * OH + oh) * OW + ix) * CHW ++ (ic * (KW * KH) + ky * KW + kx);++ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ dst[offset_dst] = 0.0f;+ } else {+ const int64_t offset_src = ic * offset_delta + batch * batch_offset;+ dst[offset_dst] = x[offset_src + iih * IW + iiw];}+}- cuda_pool_alloc(size_t size) {- alloc(size);+template <typename Ti, typename To>+static __global__ void pool2d_nchw_kernel(+ const int ih, const int iw, const int oh, const int ow,+ const int kh, const int kw, const int sh, const int sw,+ const int ph, const int pw, const int parallel_elements,+ const Ti* src, To* dst, const enum ggml_op_pool op) {+ int idx = threadIdx.x + blockIdx.x * blockDim.x;+ if (idx >= parallel_elements) {+ return;}- ~cuda_pool_alloc() {- if (ptr != nullptr) {- ggml_cuda_pool_free(device, ptr, actual_size);+ const int I_HW = ih * iw;+ const int O_HW = oh * ow;+ const int nc = idx / O_HW;+ const int cur_oh = idx % O_HW / ow;+ const int cur_ow = idx % O_HW % ow;+ const Ti* i_ptr = src + nc * I_HW;+ To* o_ptr = dst + nc * O_HW;+ const int start_h = cur_oh * sh - ph;+ const int bh = max(0, start_h);+ const int eh = min(ih, start_h + kh);+ const int start_w = cur_ow * sw - pw;+ const int bw = max(0, start_w);+ const int ew = min(iw, start_w + kw);+ const To scale = 1. / (kh * kw);+ To res = 0;++ switch (op) {+ case GGML_OP_POOL_AVG: res = 0; break;+ case GGML_OP_POOL_MAX: res = -FLT_MAX; break;+ default: assert(false);+ }++ for (int i = bh; i < eh; i += 1) {+ for (int j = bw; j < ew; j += 1) {+#if __CUDA_ARCH__ >= 350+ Ti cur = __ldg(i_ptr + i * iw + j);+#else+ Ti cur = i_ptr[i * iw + j];+#endif+ switch (op) {+ case GGML_OP_POOL_AVG: res += cur * scale; break;+ case GGML_OP_POOL_MAX: res = max(res, (To)cur); break;+ default: assert(false);+ }}}+ o_ptr[cur_oh * ow + cur_ow] = res;+}- T * get() {- return ptr;- }+template<int qk, int qr, dequantize_kernel_t dq>+static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {- cuda_pool_alloc() = default;- cuda_pool_alloc(const cuda_pool_alloc &) = delete;- cuda_pool_alloc(cuda_pool_alloc &&) = delete;- cuda_pool_alloc& operator=(const cuda_pool_alloc &) = delete;- cuda_pool_alloc& operator=(cuda_pool_alloc &&) = delete;-};+ GGML_TENSOR_BINARY_OP_LOCALS-static bool g_cublas_loaded = false;+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);+ const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);-GGML_CALL bool ggml_cublas_loaded(void) {- return g_cublas_loaded;-}+ // strides in elements+ //const size_t s0 = nb0 / ggml_element_size(dst);+ const size_t s1 = nb1 / ggml_element_size(dst);+ const size_t s2 = nb2 / ggml_element_size(dst);+ const size_t s3 = nb3 / ggml_element_size(dst);-GGML_CALL void ggml_init_cublas() {- static bool initialized = false;+ const size_t s10 = nb10 / ggml_element_size(src1);+ const size_t s11 = nb11 / ggml_element_size(src1);+ const size_t s12 = nb12 / ggml_element_size(src1);+ //const size_t s13 = nb13 / ggml_element_size(src1);- if (!initialized) {+ GGML_ASSERT(ne00 % 2 == 0);- // rocblas_initialize() will abort() if no devices exist- if (cudaGetDeviceCount(&g_device_count) != cudaSuccess || g_device_count == 0) {- initialized = true;- g_cublas_loaded = false;- return;- }+ k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10, ne11,*/ ne12, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);-#ifdef __HIP_PLATFORM_AMD__-#ifndef GGML_USE_TINYBLAS- // Workaround for a rocBLAS bug when using multiple graphics cards:- // https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1346- rocblas_initialize();-#endif- CUDA_CHECK(cudaDeviceSynchronize());-#endif+ GGML_UNUSED(dst);+}- if (cudaGetDeviceCount(&g_device_count) != cudaSuccess) {- initialized = true;- g_cublas_loaded = false;- if (!FLAG_log_disable)- fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__);- return;- }+template<typename src0_t>+static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {- GGML_ASSERT(g_device_count <= GGML_CUDA_MAX_DEVICES);- int64_t total_vram = 0;- if (!FLAG_log_disable) {-#if defined(GGML_CUDA_FORCE_MMQ)- fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__);-#else- fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: no\n", __func__);-#endif-#if defined(CUDA_USE_TENSOR_CORES)- fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: yes\n", __func__);-#else- fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: no\n", __func__);-#endif- fprintf(stderr, "%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, g_device_count);- }- for (int id = 0; id < g_device_count; ++id) {- int device_vmm = 0;+ GGML_TENSOR_BINARY_OP_LOCALS-#if !defined(GGML_USE_HIPBLAS)- CUdevice device;- CU_CHECK(cuDeviceGet(&device, id));- CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device));-- if (device_vmm) {- CUmemAllocationProp alloc_prop = {};- alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;- alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;- alloc_prop.location.id = id;- CU_CHECK(cuMemGetAllocationGranularity(&g_device_caps[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));- }-#endif // !defined(GGML_USE_HIPBLAS)- g_device_caps[id].vmm = !!device_vmm;+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);+ const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);- cudaDeviceProp prop;- CUDA_CHECK(cudaGetDeviceProperties(&prop, id));- if (!FLAG_log_disable)- fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");+ // strides in elements+ //const size_t s0 = nb0 / ggml_element_size(dst);+ const size_t s1 = nb1 / ggml_element_size(dst);+ const size_t s2 = nb2 / ggml_element_size(dst);+ const size_t s3 = nb3 / ggml_element_size(dst);- g_default_tensor_split[id] = total_vram;- total_vram += prop.totalGlobalMem;+ const size_t s10 = nb10 / ggml_element_size(src1);+ const size_t s11 = nb11 / ggml_element_size(src1);+ const size_t s12 = nb12 / ggml_element_size(src1);+ //const size_t s13 = nb13 / ggml_element_size(src1);-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- g_device_caps[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;-#else- g_device_caps[id].cc = 100*prop.major + 10*prop.minor;-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- g_device_caps[id].smpb = prop.sharedMemPerBlock;- }- for (int id = 0; id < g_device_count; ++id) {- g_default_tensor_split[id] /= total_vram;- }+ k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10, ne11,*/ ne12, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);- for (int id = 0; id < g_device_count; ++id) {- ggml_cuda_set_device(id);+ GGML_UNUSED(dst);+}- // create cuda streams- for (int is = 0; is < MAX_STREAMS; ++is) {- CUDA_CHECK(cudaStreamCreateWithFlags(&g_cudaStreams[id][is], cudaStreamNonBlocking));- }+template<float (*bin_op)(const float, const float)>+struct bin_bcast_cuda {+ template<typename src0_t, typename src1_t, typename dst_t>+ void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,+ const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,+ cudaStream_t stream) {- // create cublas handle- CUBLAS_CHECK(cublasCreate(&g_cublas_handles[id]));- CUBLAS_CHECK(cublasSetMathMode(g_cublas_handles[id], CUBLAS_TF32_TENSOR_OP_MATH));- }+ GGML_TENSOR_BINARY_OP_LOCALS- // configure logging to stdout- // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));+ int nr0 = ne10/ne0;+ int nr1 = ne11/ne1;+ int nr2 = ne12/ne2;+ int nr3 = ne13/ne3;- initialized = true;- g_cublas_loaded = true;- }-}+ int nr[4] = { nr0, nr1, nr2, nr3 };-GGML_CALL void * ggml_cuda_host_malloc(size_t size) {- if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {- return nullptr;- }+ // collapse dimensions until first broadcast dimension+ int64_t cne0[] = {ne0, ne1, ne2, ne3};+ int64_t cne1[] = {ne10, ne11, ne12, ne13};+ size_t cnb0[] = {nb0, nb1, nb2, nb3};+ size_t cnb1[] = {nb10, nb11, nb12, nb13};+ auto collapse = [](int64_t cne[]) {+ cne[0] *= cne[1];+ cne[1] = cne[2];+ cne[2] = cne[3];+ cne[3] = 1;+ };- void * ptr = nullptr;- cudaError_t err = cudaMallocHost((void **) &ptr, size);- if (err != cudaSuccess) {- // clear the error- cudaGetLastError();- fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory: %s\n",- size/1024.0/1024.0, cudaGetErrorString(err));- return nullptr;- }+ auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {+ cnb[1] *= cne[1];+ cnb[2] *= cne[2];+ cnb[3] *= cne[3];+ };- return ptr;-}+ for (int i = 0; i < 4; i++) {+ if (nr[i] != 1) {+ break;+ }+ if (i > 0) {+ collapse_nb(cnb0, cne0);+ collapse_nb(cnb1, cne1);+ collapse(cne0);+ collapse(cne1);+ }+ }+ {+ int64_t ne0 = cne0[0];+ int64_t ne1 = cne0[1];+ int64_t ne2 = cne0[2];+ int64_t ne3 = cne0[3];-GGML_CALL void ggml_cuda_host_free(void * ptr) {- CUDA_CHECK(cudaFreeHost(ptr));-}+ int64_t ne10 = cne1[0];+ int64_t ne11 = cne1[1];+ int64_t ne12 = cne1[2];+ int64_t ne13 = cne1[3];-static cudaError_t ggml_cuda_cpy_tensor_2d(- void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) {+ size_t nb0 = cnb0[0];+ size_t nb1 = cnb0[1];+ size_t nb2 = cnb0[2];+ size_t nb3 = cnb0[3];- cudaMemcpyKind kind;- char * src_ptr;- if (src->backend == GGML_BACKEND_TYPE_CPU) {- kind = cudaMemcpyHostToDevice;- src_ptr = (char *) src->data;- } else if (src->backend == GGML_BACKEND_TYPE_GPU || src->backend == GGML_BACKEND_TYPE_GPU_SPLIT) {- GGML_ASSERT(src->backend != GGML_BACKEND_TYPE_GPU_SPLIT || (i1_low == 0 && i1_high == src->ne[1]));- kind = cudaMemcpyDeviceToDevice;- ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src->extra;- int id;- CUDA_CHECK(cudaGetDevice(&id));- src_ptr = (char *) extra->data_device[id];- } else {- GGML_ASSERT(false);- }- char * dst_ptr = (char *) dst;+ size_t nb10 = cnb1[0];+ size_t nb11 = cnb1[1];+ size_t nb12 = cnb1[2];+ size_t nb13 = cnb1[3];- const int64_t ne0 = src->ne[0];- const int64_t nb0 = src->nb[0];- const int64_t nb1 = src->nb[1];- const int64_t nb2 = src->nb[2];- const int64_t nb3 = src->nb[3];- const enum ggml_type type = src->type;- const int64_t ts = ggml_type_size(type);- const int64_t bs = ggml_blck_size(type);- int64_t i1_diff = i1_high - i1_low;+ size_t s0 = nb0 / sizeof(dst_t);+ size_t s1 = nb1 / sizeof(dst_t);+ size_t s2 = nb2 / sizeof(dst_t);+ size_t s3 = nb3 / sizeof(dst_t);- const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3;- if (nb0 == ts && nb1 == ts*ne0/bs) {- return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, kind, stream);- } else if (nb0 == ts) {- return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, kind, stream);- } else {- for (int64_t i1 = 0; i1 < i1_diff; i1++) {- const void * rx = (const void *) ((const char *) x + i1*nb1);- void * rd = (void *) (dst_ptr + i1*ts*ne0/bs);- // pretend the row is a matrix with cols=1- cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, kind, stream);- if (r != cudaSuccess) return r;- }- return cudaSuccess;- }-}+ size_t s10 = nb10 / sizeof(src1_t);+ size_t s11 = nb11 / sizeof(src1_t);+ size_t s12 = nb12 / sizeof(src1_t);+ size_t s13 = nb13 / sizeof(src1_t);-static void ggml_cuda_op_get_rows(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t stream) {+ GGML_ASSERT(s0 == 1);+ GGML_ASSERT(s10 == 1);- GGML_ASSERT(src1->type == GGML_TYPE_I32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);+ const int block_size = 128;- GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));- GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));- GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));+ int64_t hne0 = std::max(ne0/2LL, 1LL);- const int32_t * src1_i32 = (const int32_t *) src1_d;+ dim3 block_dims;+ block_dims.x = std::min<unsigned int>(hne0, block_size);+ block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);+ block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);- switch (src0->type) {- case GGML_TYPE_F16:- get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_F32:- get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q4_0:- get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q4_1:- get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q5_0:- get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q5_1:- get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q8_0:- get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- default:- // TODO: k-quants- fprintf(stderr, "%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));- GGML_ASSERT(false);- break;+ dim3 block_nums(+ (hne0 + block_dims.x - 1) / block_dims.x,+ (ne1 + block_dims.y - 1) / block_dims.y,+ (ne2*ne3 + block_dims.z - 1) / block_dims.z+ );++ if (block_nums.z > 65535) {+ // this is the maximum number of blocks in z direction, fallback to 1D grid kernel+ int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;+ k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne0, ne1, ne2, ne3,+ ne10, ne11, ne12, ne13,+ /* s0, */ s1, s2, s3,+ /* s10, */ s11, s12, s13);+ } else {+ k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne0, ne1, ne2, ne3,+ ne10, ne11, ne12, ne13,+ /* s0, */ s1, s2, s3,+ /* s10, */ s11, s12, s13);+ }+ }}-}+};-template<class op>-static void ggml_cuda_op_bin_bcast(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+static void acc_f32_cuda(const float * x, const float * y, float * dst, const int n_elements,+ const int ne10, const int ne11, const int ne12,+ const int nb1, const int nb2, const int offset, cudaStream_t stream) {+ int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;+ acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset);+}- GGML_ASSERT(src1->type == GGML_TYPE_F32);+static void gelu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_GELU_BLOCK_SIZE - 1) / CUDA_GELU_BLOCK_SIZE;+ gelu_f32<<<num_blocks, CUDA_GELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- op()(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);- } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {- op()(src0, src1, dst, (const half *) src0_dd, src1_dd, (half *) dst_dd, main_stream);- } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {- op()(src0, src1, dst, (const half *) src0_dd, src1_dd, dst_dd, main_stream);- } else {- fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,- ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));- GGML_ASSERT(false);- }+static void silu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_SILU_BLOCK_SIZE - 1) / CUDA_SILU_BLOCK_SIZE;+ silu_f32<<<num_blocks, CUDA_SILU_BLOCK_SIZE, 0, stream>>>(x, dst, k);}-static void ggml_cuda_op_repeat(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t main_stream) {+static void gelu_quick_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_GELU_BLOCK_SIZE - 1) / CUDA_GELU_BLOCK_SIZE;+ gelu_quick_f32<<<num_blocks, CUDA_GELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, src0, dst, nullptr, src0_d, dst_d, main_stream);+static void tanh_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_TANH_BLOCK_SIZE - 1) / CUDA_TANH_BLOCK_SIZE;+ tanh_f32<<<num_blocks, CUDA_TANH_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}- (void) src1;- (void) src1_d;+static void relu_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;+ relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);}-static void ggml_cuda_op_add(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+static void hardsigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_HARDSIGMOID_BLOCK_SIZE - 1) / CUDA_HARDSIGMOID_BLOCK_SIZE;+ hardsigmoid_f32<<<num_blocks, CUDA_HARDSIGMOID_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+static void hardswish_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_HARDSWISH_BLOCK_SIZE - 1) / CUDA_HARDSWISH_BLOCK_SIZE;+ hardswish_f32<<<num_blocks, CUDA_HARDSWISH_BLOCK_SIZE, 0, stream>>>(x, dst, k);}-static void ggml_cuda_op_acc(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+static void leaky_relu_f32_cuda(const float * x, float * dst, const int k, const float negative_slope, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_RELU_BLOCK_SIZE - 1) / CUDA_RELU_BLOCK_SIZE;+ leaky_relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k, negative_slope);+}- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);- GGML_ASSERT(dst->ne[3] == 1); // just 3D tensors supported+static void sqr_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_SQR_BLOCK_SIZE - 1) / CUDA_SQR_BLOCK_SIZE;+ sqr_f32<<<num_blocks, CUDA_SQR_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}- int nb1 = dst->op_params[0] / 4; // 4 bytes of float32- int nb2 = dst->op_params[1] / 4; // 4 bytes of float32- // int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused- int offset = dst->op_params[3] / 4; // offset in bytes+static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {+ GGML_ASSERT(ncols % WARP_SIZE == 0);+ if (ncols < 1024) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ } else {+ const dim3 block_dims(1024, 1, 1);+ norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ }+}- acc_f32_cuda(src0_dd, src1_dd, dst_dd, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, main_stream);+static void group_norm_f32_cuda(const float * x, float * dst, const int num_groups, const int group_size, const int ne_elements, cudaStream_t stream) {+ static const float eps = 1e-6f;+ if (group_size < 1024) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);+ } else {+ const dim3 block_dims(1024, 1, 1);+ group_norm_f32<1024><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);+ }+}- (void) dst;+static void concat_f32_cuda(const float * x, const float * y, float * dst, const int ne0, int ne1, int ne2, int ne02, cudaStream_t stream) {+ int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;+ dim3 gridDim(num_blocks, ne1, ne2);+ concat_f32<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);}-static void ggml_cuda_op_mul(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+static void upscale_f32_cuda(const float * x, float * dst, const int ne00, const int ne01, const int ne02, const int ne03,+ const int scale_factor, cudaStream_t stream) {+ int ne0 = (ne00 * scale_factor);+ int num_blocks = (ne0 + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;+ dim3 gridDim(num_blocks, (ne01 * scale_factor), ne02*ne03);+ upscale_f32<<<gridDim, CUDA_UPSCALE_BLOCK_SIZE, 0, stream>>>(x, dst, ne00, ne00 * ne01, scale_factor);+}- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+static void pad_f32_cuda(const float * x, float * dst,+ const int ne00, const int ne01, const int ne02, const int ne03,+ const int ne0, const int ne1, const int ne2, const int ne3, cudaStream_t stream) {+ int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE;+ dim3 gridDim(num_blocks, ne1, ne2*ne3);+ pad_f32<<<gridDim, CUDA_PAD_BLOCK_SIZE, 0, stream>>>(x, dst, ne0, ne00, ne01, ne02, ne03);}-static void ggml_cuda_op_div(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+static void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {+ int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;+ arange_f32<<<num_blocks, CUDA_ARANGE_BLOCK_SIZE, 0, stream>>>(dst, ne0, start, step);+}- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+static void timestep_embedding_f32_cuda(const float * x, float * dst, const int ne00, const int nb1,+ const int dim, const int max_period, cudaStream_t stream) {+ int half_ceil = (dim + 1) / 2;+ int num_blocks = (half_ceil + CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE - 1) / CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE;+ dim3 gridDim(num_blocks, ne00, 1);+ timestep_embedding_f32<<<gridDim, CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE, 0, stream>>>(x, dst, nb1, dim, max_period);}-static void ggml_cuda_op_gelu(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {+ GGML_ASSERT(ncols % WARP_SIZE == 0);+ if (ncols < 1024) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ rms_norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ } else {+ const dim3 block_dims(1024, 1, 1);+ rms_norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);+ }+}- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+static void quantize_row_q8_1_cuda(const float * x, void * vy, const int kx, const int ky, const int kx_padded, cudaStream_t stream) {+ const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;+ const dim3 num_blocks(block_num_x, ky, 1);+ const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);+ quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);+}- gelu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {+ const int num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);+ dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);+}- (void) src1;- (void) dst;- (void) src1_dd;+static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;+ if (k % CUDA_Q8_0_NE_ALIGN == 0) {+ const bool need_check = false;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ } else {+ const bool need_check = true;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ }}-static void ggml_cuda_op_silu(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+template<typename dst_t>+static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);+#else+ dequantize_block_q2_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+template<typename dst_t>+static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);+#else+ dequantize_block_q3_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- silu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);+template<typename dst_t>+static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);+}- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);}-static void ggml_cuda_op_gelu_quick(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+template<typename dst_t>+static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);+}- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+template<typename dst_t>+static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);+#else+ dequantize_block_q5_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- gelu_quick_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);+template<typename dst_t>+static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);+#else+ dequantize_block_q6_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);}-static void ggml_cuda_op_tanh(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- tanh_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;-}--static void ggml_cuda_op_relu(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- relu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;-}--static void ggml_cuda_op_hardsigmoid(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- hardsigmoid_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;-}--static void ggml_cuda_op_hardswish(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- hardswish_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;-}--static void ggml_cuda_op_leaky_relu(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- float negative_slope;- memcpy(&negative_slope, dst->op_params, sizeof(float));-- leaky_relu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), negative_slope, main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);}-static void ggml_cuda_op_sqr(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- sqr_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);}-static void ggml_cuda_op_norm(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- const int64_t ne00 = src0->ne[0];- const int64_t nrows = ggml_nrows(src0);-- float eps;- memcpy(&eps, dst->op_params, sizeof(float));-- norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, eps, main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);}-static void ggml_cuda_op_group_norm(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- int num_groups = dst->op_params[0];- int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);- group_norm_f32_cuda(src0_dd, dst_dd, num_groups, group_size, src0->ne[0] * src0->ne[1] * src0->ne[2], main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);}-static void ggml_cuda_op_concat(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);-- for (int i3 = 0; i3 < dst->ne[3]; i3++) {- concat_f32_cuda(src0_dd + i3 * (src0->nb[3] / 4), src1_dd + i3 * (src1->nb[3] / 4), dst_dd + i3 * (dst->nb[3] / 4), dst->ne[0], dst->ne[1], dst->ne[2], src0->ne[2], main_stream);- }-- (void) src1;- (void) dst;+template<typename dst_t>+static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);}-static void ggml_cuda_op_upscale(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);- GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors-- const int scale_factor = dst->op_params[0];-- upscale_f32_cuda(src0_dd, dst_dd, src0->ne[0], src0->ne[1], src0->ne[2], scale_factor, main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = (k + QK_K - 1) / QK_K;+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);}-static void ggml_cuda_op_pad(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);- GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors-- pad_f32_cuda(src0_dd, dst_dd,- src0->ne[0], src0->ne[1], src0->ne[2],- dst->ne[0], dst->ne[1], dst->ne[2], main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;+template<typename dst_t>+static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = (k + QK_K - 1) / QK_K;+#if QK_K == 64+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);+#else+ dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);+#endif}-static void ggml_cuda_op_rms_norm(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);-- const int64_t ne00 = src0->ne[0];- const int64_t nrows = ggml_nrows(src0);-- float eps;- memcpy(&eps, dst->op_params, sizeof(float));-- rms_norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, eps, main_stream);-- (void) src1;- (void) dst;- (void) src1_dd;+template <typename src_t, typename dst_t>+static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;+ convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);}-static void ggml_cuda_op_mul_mat_q(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream) {-- const int64_t ne00 = src0->ne[0];-- const int64_t ne10 = src1->ne[0];- GGML_ASSERT(ne10 % QK8_1 == 0);-- const int64_t ne0 = dst->ne[0];-- const int64_t row_diff = row_high - row_low;-+static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {int id;- CUDA_CHECK(cudaGetDevice(&id));-- // the main device has a larger memory buffer to hold the results from all GPUs- // nrows_dst == nrows of the matrix that the kernel writes into- const int64_t nrows_dst = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff;-- switch (src0->type) {+ switch (type) {case GGML_TYPE_Q4_0:- ggml_mul_mat_q4_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_row_q4_0_cuda;case GGML_TYPE_Q4_1:- ggml_mul_mat_q4_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_row_q4_1_cuda;case GGML_TYPE_Q5_0:- ggml_mul_mat_q5_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;case GGML_TYPE_Q5_1:- ggml_mul_mat_q5_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;case GGML_TYPE_Q8_0:- ggml_mul_mat_q8_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ CUDA_CHECK(cudaGetDevice(&id));+ if (get_cuda_global_info().devices[id].cc >= CC_PASCAL) {+ return dequantize_block_q8_0_f16_cuda;+ }+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;case GGML_TYPE_Q2_K:- ggml_mul_mat_q2_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_row_q2_K_cuda;case GGML_TYPE_Q3_K:- ggml_mul_mat_q3_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_row_q3_K_cuda;case GGML_TYPE_Q4_K:- ggml_mul_mat_q4_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_row_q4_K_cuda;case GGML_TYPE_Q5_K:- ggml_mul_mat_q5_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_row_q5_K_cuda;case GGML_TYPE_Q6_K:- ggml_mul_mat_q6_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);- break;+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ2_S:+ return dequantize_row_iq2_s_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;+ case GGML_TYPE_IQ4_XS:+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;+ case GGML_TYPE_F32:+ return convert_unary_cuda<float>;default:- GGML_ASSERT(false);- break;+ return nullptr;}-- (void) src1;- (void) dst;- (void) src1_ddf_i;}-static int64_t get_row_rounding(ggml_type type, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split) {- int64_t min_compute_capability = INT_MAX;- int64_t max_compute_capability = INT_MIN;- for (int id = 0; id < g_device_count; ++id) {- if (tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) {- if (min_compute_capability > g_device_caps[id].cc) {- min_compute_capability = g_device_caps[id].cc;- }- if (max_compute_capability < g_device_caps[id].cc) {- max_compute_capability = g_device_caps[id].cc;- }- }- }--#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- switch(type) {+static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {+ switch (type) {case GGML_TYPE_Q4_0:+ return dequantize_row_q4_0_cuda;case GGML_TYPE_Q4_1:+ return dequantize_row_q4_1_cuda;case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;case GGML_TYPE_Q8_0:- return max_compute_capability >= CC_RDNA2 ? 128 : 64;- case GGML_TYPE_F16:- case GGML_TYPE_F32:- return 1;+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;case GGML_TYPE_Q2_K:- return max_compute_capability >= CC_RDNA2 ? 128 : 32;+ return dequantize_row_q2_K_cuda;case GGML_TYPE_Q3_K:- return min_compute_capability < CC_RDNA2 ? 128 : 64;+ return dequantize_row_q3_K_cuda;case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;case GGML_TYPE_IQ2_S:+ return dequantize_row_iq2_s_cuda;case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;case GGML_TYPE_IQ4_XS:+ return dequantize_row_iq4_xs_cuda;case GGML_TYPE_IQ3_S:- return max_compute_capability >= CC_RDNA2 ? 128 : 64;- default:- GGML_ASSERT(false);- }-#else- switch(type) {- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- return max_compute_capability >= CC_VOLTA ? 128 : 64;- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- return 64;+ return dequantize_row_iq3_s_cuda;case GGML_TYPE_F16:- case GGML_TYPE_F32:- return 1;- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_IQ2_XXS:- case GGML_TYPE_IQ2_XS:- case GGML_TYPE_IQ2_S:- case GGML_TYPE_IQ3_XXS:- case GGML_TYPE_IQ1_S:- case GGML_TYPE_IQ4_NL:- case GGML_TYPE_IQ4_XS:- case GGML_TYPE_IQ3_S:- return max_compute_capability >= CC_VOLTA ? 128 : 64;- case GGML_TYPE_Q6_K:- return 64;+ return convert_unary_cuda<half>;default:- GGML_ASSERT(false);+ return nullptr;}-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)}-static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split, int id) {- const int64_t nrows = ggml_nrows(tensor);- const int64_t rounding = get_row_rounding(tensor->type, tensor_split);+static void dequantize_mul_mat_vec_q4_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}- *row_low = id == 0 ? 0 : nrows*tensor_split[id];- *row_low -= *row_low % rounding;+static void dequantize_mul_mat_vec_q4_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}- if (id == g_device_count - 1) {- *row_high = nrows;- } else {- *row_high = nrows*tensor_split[id + 1];- *row_high -= *row_high % rounding;- }+static void dequantize_mul_mat_vec_q5_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-static void ggml_cuda_op_mul_mat_vec_q(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream) {+static void dequantize_mul_mat_vec_q5_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}- const int64_t ne00 = src0->ne[0];- const int64_t row_diff = row_high - row_low;+static void dequantize_mul_mat_vec_q8_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}- const int64_t ne10 = src1->ne[0];- GGML_ASSERT(ne10 % QK8_1 == 0);+static void dequantize_mul_mat_vec_q2_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q2_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}- const int64_t ne0 = dst->ne[0];+static void dequantize_mul_mat_vec_q3_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2 / K_QUANTS_PER_ITERATION;+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q3_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}++static void dequantize_mul_mat_vec_q4_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2 / K_QUANTS_PER_ITERATION;+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q4_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}++static void dequantize_mul_mat_vec_q5_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const dim3 block_dims(32, 1, 1);+ dequantize_mul_mat_vec_q5_k<<<nrows, block_dims, 0, stream>>>(vx, y, dst, ncols);+}++static void dequantize_mul_mat_vec_q6_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2 / K_QUANTS_PER_ITERATION;+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q6_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}++static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<1, 1, convert_f16>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+}++template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot>+static void mul_mat_vec_q_cuda(+ const void * vx, const void * vy, float * dst,+ const int ncols_x, const int nrows_x, const int nrows_y, const int ncols_y, const int nrows_dst, cudaStream_t stream) {++ GGML_ASSERT(ncols_x % qk == 0);+ GGML_ASSERT(ncols_y <= MMVQ_MAX_BATCH_SIZE);int id;CUDA_CHECK(cudaGetDevice(&id));- // the main device has a larger memory buffer to hold the results from all GPUs- // nrows_dst == nrows of the matrix that the kernel writes into- const int64_t nrows_dst = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff;+ int64_t nwarps = 1;+ int64_t rows_per_cuda_block = 1;- switch (src0->type) {- case GGML_TYPE_Q4_0:- mul_mat_vec_q_cuda<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_Q4_1:- mul_mat_vec_q_cuda<QK4_1, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ if (get_cuda_global_info().devices[id].cc < CC_RDNA2) { // NVIDIA and AMD older than RDNA2+ switch(ncols_y) {+ case 1:+ nwarps = 4;+ rows_per_cuda_block = 1;+ break;+ case 2:+ case 3:+ case 4:+ nwarps = 4;+ rows_per_cuda_block = 2;+ break;+ case 5:+ case 6:+ case 7:+ case 8:+ nwarps = 2;+ rows_per_cuda_block = 2;+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ }+ const int64_t nblocks = (nrows_x + rows_per_cuda_block - 1) / rows_per_cuda_block;+ const dim3 block_nums(nblocks, 1, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ switch (ncols_y) {+ case 1:+ mul_mat_vec_q<1, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;- case GGML_TYPE_Q5_0:- mul_mat_vec_q_cuda<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ case 2:+ mul_mat_vec_q<2, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;- case GGML_TYPE_Q5_1:- mul_mat_vec_q_cuda<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ case 3:+ mul_mat_vec_q<3, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;- case GGML_TYPE_Q8_0:- mul_mat_vec_q_cuda<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ case 4:+ mul_mat_vec_q<4, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;- case GGML_TYPE_Q2_K:- mul_mat_vec_q_cuda<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ case 5:+ mul_mat_vec_q<5, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;- case GGML_TYPE_Q3_K:- mul_mat_vec_q_cuda<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ case 6:+ mul_mat_vec_q<6, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;- case GGML_TYPE_Q4_K:- mul_mat_vec_q_cuda<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ case 7:+ mul_mat_vec_q<7, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;- case GGML_TYPE_Q5_K:- mul_mat_vec_q_cuda<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_Q6_K:- mul_mat_vec_q_cuda<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ2_XXS:- mul_mat_vec_q_cuda<QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ2_XS:- mul_mat_vec_q_cuda<QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ2_S:- mul_mat_vec_q_cuda<QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ3_XXS:- mul_mat_vec_q_cuda<QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ1_S:- mul_mat_vec_q_cuda<QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ4_NL:- mul_mat_vec_q_cuda<QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ4_XS:- mul_mat_vec_q_cuda<QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);- break;- case GGML_TYPE_IQ3_S:- mul_mat_vec_q_cuda<QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>- (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ case 8:+ mul_mat_vec_q<8, qk, qi, block_q_t, vdr, vec_dot>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, nrows_y, nrows_dst);break;default:GGML_ASSERT(false);break;}+}++static void ggml_mul_mat_q4_0_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q4_0_RDNA2;+ mmq_y = MMQ_Y_Q4_0_RDNA2;+ nwarps = NWARPS_Q4_0_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q4_0_RDNA1;+ mmq_y = MMQ_Y_Q4_0_RDNA1;+ nwarps = NWARPS_Q4_0_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q4_0_AMPERE;+ mmq_y = MMQ_Y_Q4_0_AMPERE;+ nwarps = NWARPS_Q4_0_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q4_0_PASCAL;+ mmq_y = MMQ_Y_Q4_0_PASCAL;+ nwarps = NWARPS_Q4_0_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q4_1_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q4_1_RDNA2;+ mmq_y = MMQ_Y_Q4_1_RDNA2;+ nwarps = NWARPS_Q4_1_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q4_1_RDNA1;+ mmq_y = MMQ_Y_Q4_1_RDNA1;+ nwarps = NWARPS_Q4_1_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q4_1_AMPERE;+ mmq_y = MMQ_Y_Q4_1_AMPERE;+ nwarps = NWARPS_Q4_1_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q4_1_PASCAL;+ mmq_y = MMQ_Y_Q4_1_PASCAL;+ nwarps = NWARPS_Q4_1_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q5_0_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q5_0_RDNA2;+ mmq_y = MMQ_Y_Q5_0_RDNA2;+ nwarps = NWARPS_Q5_0_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q5_0_RDNA1;+ mmq_y = MMQ_Y_Q5_0_RDNA1;+ nwarps = NWARPS_Q5_0_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q5_0_AMPERE;+ mmq_y = MMQ_Y_Q5_0_AMPERE;+ nwarps = NWARPS_Q5_0_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q5_0_PASCAL;+ mmq_y = MMQ_Y_Q5_0_PASCAL;+ nwarps = NWARPS_Q5_0_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q5_1_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q5_1_RDNA2;+ mmq_y = MMQ_Y_Q5_1_RDNA2;+ nwarps = NWARPS_Q5_1_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q5_1_RDNA1;+ mmq_y = MMQ_Y_Q5_1_RDNA1;+ nwarps = NWARPS_Q5_1_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q5_1_AMPERE;+ mmq_y = MMQ_Y_Q5_1_AMPERE;+ nwarps = NWARPS_Q5_1_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q5_1_PASCAL;+ mmq_y = MMQ_Y_Q5_1_PASCAL;+ nwarps = NWARPS_Q5_1_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q8_0_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q8_0_RDNA2;+ mmq_y = MMQ_Y_Q8_0_RDNA2;+ nwarps = NWARPS_Q8_0_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q8_0_RDNA1;+ mmq_y = MMQ_Y_Q8_0_RDNA1;+ nwarps = NWARPS_Q8_0_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q8_0_AMPERE;+ mmq_y = MMQ_Y_Q8_0_AMPERE;+ nwarps = NWARPS_Q8_0_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q8_0_PASCAL;+ mmq_y = MMQ_Y_Q8_0_PASCAL;+ nwarps = NWARPS_Q8_0_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q2_K_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q2_K_RDNA2;+ mmq_y = MMQ_Y_Q2_K_RDNA2;+ nwarps = NWARPS_Q2_K_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q2_K_RDNA1;+ mmq_y = MMQ_Y_Q2_K_RDNA1;+ nwarps = NWARPS_Q2_K_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q2_K_AMPERE;+ mmq_y = MMQ_Y_Q2_K_AMPERE;+ nwarps = NWARPS_Q2_K_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q2_K_PASCAL;+ mmq_y = MMQ_Y_Q2_K_PASCAL;+ nwarps = NWARPS_Q2_K_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q3_K_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++#if QK_K == 256++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q3_K_RDNA2;+ mmq_y = MMQ_Y_Q3_K_RDNA2;+ nwarps = NWARPS_Q3_K_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q3_K_RDNA1;+ mmq_y = MMQ_Y_Q3_K_RDNA1;+ nwarps = NWARPS_Q3_K_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q3_K_AMPERE;+ mmq_y = MMQ_Y_Q3_K_AMPERE;+ nwarps = NWARPS_Q3_K_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q3_K_PASCAL;+ mmq_y = MMQ_Y_Q3_K_PASCAL;+ nwarps = NWARPS_Q3_K_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+#endif+}++static void ggml_mul_mat_q4_K_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q4_K_RDNA2;+ mmq_y = MMQ_Y_Q4_K_RDNA2;+ nwarps = NWARPS_Q4_K_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q4_K_RDNA1;+ mmq_y = MMQ_Y_Q4_K_RDNA1;+ nwarps = NWARPS_Q4_K_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q4_K_AMPERE;+ mmq_y = MMQ_Y_Q4_K_AMPERE;+ nwarps = NWARPS_Q4_K_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q4_K_PASCAL;+ mmq_y = MMQ_Y_Q4_K_PASCAL;+ nwarps = NWARPS_Q4_K_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q5_K_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q5_K_RDNA2;+ mmq_y = MMQ_Y_Q5_K_RDNA2;+ nwarps = NWARPS_Q5_K_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q5_K_RDNA1;+ mmq_y = MMQ_Y_Q5_K_RDNA1;+ nwarps = NWARPS_Q5_K_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q5_K_AMPERE;+ mmq_y = MMQ_Y_Q5_K_AMPERE;+ nwarps = NWARPS_Q5_K_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q5_K_PASCAL;+ mmq_y = MMQ_Y_Q5_K_PASCAL;+ nwarps = NWARPS_Q5_K_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_q6_K_q8_1_cuda(+ const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,+ const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {++ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ const int compute_capability = get_cuda_global_info().devices[id].cc;++ int mmq_x, mmq_y, nwarps;+ if (compute_capability >= CC_RDNA2) {+ mmq_x = MMQ_X_Q6_K_RDNA2;+ mmq_y = MMQ_Y_Q6_K_RDNA2;+ nwarps = NWARPS_Q6_K_RDNA2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ mmq_x = MMQ_X_Q6_K_RDNA1;+ mmq_y = MMQ_Y_Q6_K_RDNA1;+ nwarps = NWARPS_Q6_K_RDNA1;+ } else if (compute_capability >= CC_VOLTA) {+ mmq_x = MMQ_X_Q6_K_AMPERE;+ mmq_y = MMQ_Y_Q6_K_AMPERE;+ nwarps = NWARPS_Q6_K_AMPERE;+ } else if (compute_capability >= MIN_CC_DP4A) {+ mmq_x = MMQ_X_Q6_K_PASCAL;+ mmq_y = MMQ_Y_Q6_K_PASCAL;+ nwarps = NWARPS_Q6_K_PASCAL;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;+ const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, nwarps, 1);++ if (nrows_x % mmq_y == 0) {+ const bool need_check = false;+ mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ } else {+ const bool need_check = true;+ mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ }+}++static void ggml_mul_mat_p021_f16_f32_cuda(+ const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x,+ const int nchannels_x, const int nchannels_y, cudaStream_t stream) {++ const dim3 block_nums(1, nrows_x, nchannels_y);+ const dim3 block_dims(WARP_SIZE, 1, 1);+ mul_mat_p021_f16_f32<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols_x, nrows_x, nchannels_x, nchannels_y);+}++static void ggml_mul_mat_vec_nc_f16_f32_cuda(+ const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x, const int row_stride_x,+ const int nchannels_x, const int nchannels_y, const int channel_stride_x, cudaStream_t stream) {++ const dim3 block_nums(1, nrows_x, nchannels_y);+ const dim3 block_dims(WARP_SIZE, 1, 1);+ mul_mat_vec_nc_f16_f32<<<block_nums, block_dims, 0, stream>>>+ (vx, y, dst, ncols_x, nrows_x, row_stride_x, channel_stride_x, nchannels_y/nchannels_x);+}+++static void ggml_cpy_f16_f32_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f16_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_f32_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f32_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_f16_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f32_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_q8_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ GGML_ASSERT(ne % QK8_0 == 0);+ const int num_blocks = ne / QK8_0;+ cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_q4_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ GGML_ASSERT(ne % QK4_0 == 0);+ const int num_blocks = ne / QK4_0;+ cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_q4_1_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ GGML_ASSERT(ne % QK4_1 == 0);+ const int num_blocks = ne / QK4_1;+ cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_q5_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ GGML_ASSERT(ne % QK5_0 == 0);+ const int num_blocks = ne / QK5_0;+ cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_q5_1_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ GGML_ASSERT(ne % QK5_1 == 0);+ const int num_blocks = ne / QK5_1;+ cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}++static void ggml_cpy_f32_iq4_nl_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- (void) src1;- (void) dst;- (void) src1_ddf_i;- (void) src1_ncols;- (void) src1_padded_row_size;+ GGML_ASSERT(ne % QK4_NL == 0);+ const int num_blocks = ne / QK4_NL;+ cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}-static void ggml_cuda_op_dequantize_mul_mat_vec(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream) {+static void ggml_cpy_f16_f16_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- const int64_t ne00 = src0->ne[0];- const int64_t row_diff = row_high - row_low;+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f16_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}- GGML_ASSERT(src1->type == GGML_TYPE_F32);- // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics-#ifdef GGML_CUDA_F16- cuda_pool_alloc<half> src1_dfloat_a;- half * src1_dfloat = nullptr; // dfloat == half- bool src1_convert_f16 =- src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||- src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||- src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;+static void scale_f32_cuda(const float * x, float * dst, const float scale, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;+ scale_f32<<<num_blocks, CUDA_SCALE_BLOCK_SIZE, 0, stream>>>(x, dst, scale, k);+}- if (src1_convert_f16) {- src1_dfloat = src1_dfloat_a.alloc(ne00);- const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);- GGML_ASSERT(to_fp16_cuda != nullptr);- to_fp16_cuda(src1_ddf_i, src1_dfloat, ne00, stream);- }-#else- const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion-#endif // GGML_CUDA_F16+static void clamp_f32_cuda(const float * x, float * dst, const float min, const float max, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE;+ clamp_f32<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, k);+}- switch (src0->type) {- case GGML_TYPE_Q4_0:- dequantize_mul_mat_vec_q4_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q4_1:- dequantize_mul_mat_vec_q4_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q5_0:- dequantize_mul_mat_vec_q5_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q5_1:- dequantize_mul_mat_vec_q5_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q8_0:- dequantize_mul_mat_vec_q8_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q2_K:- dequantize_mul_mat_vec_q2_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q3_K:- dequantize_mul_mat_vec_q3_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q4_K:- dequantize_mul_mat_vec_q4_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q5_K:- dequantize_mul_mat_vec_q5_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q6_K:- dequantize_mul_mat_vec_q6_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_F16:- convert_mul_mat_vec_f16_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- default:- GGML_ASSERT(false);- break;+template<typename T>+static void rope_cuda(+ const T * x, T * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream+) {+ GGML_ASSERT(ncols % 2 == 0);+ const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);+ const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);+ const dim3 block_nums(nrows, num_blocks_x, 1);+ if (pos == nullptr) {+ rope<T, false><<<block_nums, block_dims, 0, stream>>>(+ x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims+ );+ } else {+ rope<T, true><<<block_nums, block_dims, 0, stream>>>(+ x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims+ );}-- (void) src1;- (void) dst;- (void) src1_ddq_i;- (void) src1_ncols;- (void) src1_padded_row_size;}-static void ggml_cuda_op_mul_mat_cublas(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream) {+template<typename T>+static void rope_neox_cuda(+ const T * x, T * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,+ float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream+) {+ GGML_ASSERT(ncols % 2 == 0);+ const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);+ const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);+ const dim3 block_nums(nrows, num_blocks_x, 1);- GGML_ASSERT(src0_dd_i != nullptr);- GGML_ASSERT(src1_ddf_i != nullptr);- GGML_ASSERT(dst_dd_i != nullptr);+ const float theta_scale = powf(freq_base, -2.0f/n_dims);+ const float inv_ndims = -1.0f / n_dims;- const int64_t ne00 = src0->ne[0];- const int64_t ne10 = src1->ne[0];+ if (pos == nullptr) {+ rope_neox<T, false><<<block_nums, block_dims, 0, stream>>>(+ x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,+ theta_scale, inv_ndims+ );+ } else {+ rope_neox<T, true><<<block_nums, block_dims, 0, stream>>>(+ x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,+ theta_scale, inv_ndims+ );+ }+}- const int64_t ne0 = dst->ne[0];+static void rope_glm_f32_cuda(+ const float * x, float * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,+ float freq_base, int n_ctx, cudaStream_t stream+) {+ GGML_ASSERT(ncols % 4 == 0);+ const dim3 block_dims(CUDA_ROPE_BLOCK_SIZE/4, 1, 1);+ const int num_blocks_x = (ncols + CUDA_ROPE_BLOCK_SIZE - 1) / CUDA_ROPE_BLOCK_SIZE;+ const dim3 block_nums(num_blocks_x, nrows, 1);+ rope_glm_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, n_ctx);+}- const int64_t row_diff = row_high - row_low;+static void alibi_f32_cuda(const float * x, float * dst, const int ncols, const int nrows,+ const int k_rows, const int n_heads_log2_floor, const float m0,+ const float m1, cudaStream_t stream) {+ const dim3 block_dims(CUDA_ALIBI_BLOCK_SIZE, 1, 1);+ const int num_blocks_x = (ncols + CUDA_ALIBI_BLOCK_SIZE - 1) / (CUDA_ALIBI_BLOCK_SIZE);+ const dim3 block_nums(num_blocks_x, nrows, 1);+ alibi_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, k_rows, n_heads_log2_floor, m0, m1);+}- int id;- CUDA_CHECK(cudaGetDevice(&id));+static void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ const dim3 block_dims(WARP_SIZE, 1, 1);+ const dim3 block_nums(nrows, 1, 1);+ k_sum_rows_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);+}- // the main device has a larger memory buffer to hold the results from all GPUs- // ldc == nrows of the matrix that cuBLAS writes into- int ldc = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device ? ne0 : row_diff;+static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {+ // bitonic sort requires ncols to be power of 2+ GGML_ASSERT((ncols & (ncols - 1)) == 0);- const int compute_capability = g_device_caps[id].cc;+ const dim3 block_dims(ncols, 1, 1);+ const dim3 block_nums(1, nrows, 1);+ if (order == GGML_SORT_ORDER_ASC) {+ k_argsort_f32_i32<GGML_SORT_ORDER_ASC><<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);+ } else if (order == GGML_SORT_ORDER_DESC) {+ k_argsort_f32_i32<GGML_SORT_ORDER_DESC><<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);+ } else {+ GGML_ASSERT(false);+ }+}- if (compute_capability >= CC_VOLTA && (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) {- //printf("this branch\n");- // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32- cuda_pool_alloc<half> src0_as_f16;- if (src0->type != GGML_TYPE_F16) {- const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type);- GGML_ASSERT(to_fp16_cuda != nullptr);- size_t ne = row_diff*ne00;- src0_as_f16.alloc(ne);- to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream);- }- const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get();+static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {+ const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);+ const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;+ const dim3 block_nums(nrows_x, block_num_x, 1);+ diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);+}- cuda_pool_alloc<half> src1_as_f16;- if (src1->type != GGML_TYPE_F16) {- const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);- GGML_ASSERT(to_fp16_cuda != nullptr);- size_t ne = src1_ncols*ne10;- src1_as_f16.alloc(ne);- to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream);- }- const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get();- cuda_pool_alloc<half> dst_f16(row_diff*src1_ncols);+static void soft_max_f32_cuda(const float * x, const float * mask, const float * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {+ int nth = WARP_SIZE;+ while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;+ const dim3 block_dims(nth, 1, 1);+ const dim3 block_nums(nrows_x, 1, 1);+ const size_t shmem = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE)*sizeof(float);+ static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted.");- const half alpha_f16 = 1.0f;- const half beta_f16 = 0.0f;+ const uint32_t n_head_kv = nrows_x/nrows_y;+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv));- CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], stream));- CUBLAS_CHECK(- cublasGemmEx(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N,- row_diff, src1_ncols, ne10,- &alpha_f16, src0_ptr, CUDA_R_16F, ne00,- src1_ptr, CUDA_R_16F, ne10,- &beta_f16, dst_f16.get(), CUDA_R_16F, ldc,- CUBLAS_COMPUTE_16F,- CUBLAS_GEMM_DEFAULT_TENSOR_OP));+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);- const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);- to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);+ if (shmem < get_cuda_global_info().devices[ggml_cuda_get_device()].smpb) {+ switch (ncols_x) {+ case 32:+ soft_max_f32<true, 32, 32><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ case 64:+ soft_max_f32<true, 64, 64><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ case 128:+ soft_max_f32<true, 128, 128><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ case 256:+ soft_max_f32<true, 256, 256><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ case 512:+ soft_max_f32<true, 512, 512><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ case 1024:+ soft_max_f32<true, 1024, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ case 2048:+ soft_max_f32<true, 2048, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ case 4096:+ soft_max_f32<true, 4096, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ default:+ soft_max_f32<true, 0, 0><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ break;+ }} else {- cuda_pool_alloc<float> src0_ddq_as_f32;- cuda_pool_alloc<float> src1_ddq_as_f32;+ const size_t shmem_low = WARP_SIZE*sizeof(float);+ soft_max_f32<false, 0, 0><<<block_nums, block_dims, shmem_low, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ }+}++template <typename T>+static void im2col_cuda(const float * x, T* dst,+ int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,+ int64_t batch, int64_t batch_offset, int64_t offset_delta,+ int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {+ const int parallel_elements = OW * KW * KH;+ const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;+ dim3 block_nums(num_blocks, OH, batch * IC);+ im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);+}- if (src0->type != GGML_TYPE_F32) {- const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type);- GGML_ASSERT(to_fp32_cuda != nullptr);- src0_ddq_as_f32.alloc(row_diff*ne00);- to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream);- }- if (src1->type != GGML_TYPE_F32) {- const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type);- GGML_ASSERT(to_fp32_cuda != nullptr);- src1_ddq_as_f32.alloc(src1_ncols*ne10);- to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream);- }+static cudaError_t ggml_cuda_cpy_tensor_2d(+ void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) {- const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get();- const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src->buffer));+ char * src_ptr = (char *) src->data;+ char * dst_ptr = (char *) dst;- const float alpha = 1.0f;- const float beta = 0.0f;+ const int64_t ne0 = src->ne[0];+ const int64_t nb0 = src->nb[0];+ const int64_t nb1 = src->nb[1];+ const int64_t nb2 = src->nb[2];+ const int64_t nb3 = src->nb[3];+ const enum ggml_type type = src->type;+ const int64_t ts = ggml_type_size(type);+ const int64_t bs = ggml_blck_size(type);+ int64_t i1_diff = i1_high - i1_low;- CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], stream));- CUBLAS_CHECK(- cublasSgemm(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N,- row_diff, src1_ncols, ne10,- &alpha, src0_ddf_i, ne00,- src1_ddf1_i, ne10,- &beta, dst_dd_i, ldc));+ const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3;+ if (nb0 == ts && nb1 == ts*ne0/bs) {+ return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, cudaMemcpyDeviceToDevice, stream);+ } else if (nb0 == ts) {+ return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, cudaMemcpyDeviceToDevice, stream);+ } else {+ for (int64_t i1 = 0; i1 < i1_diff; i1++) {+ const void * rx = (const void *) ((const char *) x + i1*nb1);+ void * rd = (void *) (dst_ptr + i1*ts*ne0/bs);+ // pretend the row is a matrix with cols=1+ cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, cudaMemcpyDeviceToDevice, stream);+ if (r != cudaSuccess) {+ return r;+ }+ }+ return cudaSuccess;}-- (void) dst;- (void) src1_ddq_i;- (void) src1_padded_row_size;}-static void ggml_cuda_op_rope(+static void ggml_cuda_op_get_rows(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-- GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);- GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);- GGML_ASSERT(src0->type == dst->type);-- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne2 = dst->ne[2];- const int64_t nrows = ggml_nrows(src0);-- //const int n_past = ((int32_t *) dst->op_params)[0];- const int n_dims = ((int32_t *) dst->op_params)[1];- const int mode = ((int32_t *) dst->op_params)[2];- const int n_ctx = ((int32_t *) dst->op_params)[3];- const int n_orig_ctx = ((int32_t *) dst->op_params)[4];+ const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t stream) {- // RoPE alteration for extended context- float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;- memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));- memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));- memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));- memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));- memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));- memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));+ GGML_UNUSED(ctx);- const int32_t * pos = nullptr;- if ((mode & 1) == 0) {- GGML_ASSERT(src1->type == GGML_TYPE_I32);- GGML_ASSERT(src1->ne[0] == ne2);- pos = (const int32_t *) src1_dd;- }+ GGML_ASSERT(src1->type == GGML_TYPE_I32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);- const bool is_neox = mode & 2;- const bool is_glm = mode & 4;+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));+ GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));+ GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));- rope_corr_dims corr_dims;- ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims.v);+ const int32_t * src1_i32 = (const int32_t *) src1_d;- // compute- if (is_glm) {- GGML_ASSERT(false);- rope_glm_f32_cuda(src0_dd, dst_dd, ne00, nrows, pos, freq_scale, ne01, freq_base, n_ctx, main_stream);- } else if (is_neox) {- if (src0->type == GGML_TYPE_F32) {- rope_neox_cuda(- (const float *)src0_dd, (float *)dst_dd, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,- attn_factor, corr_dims, main_stream- );- } else if (src0->type == GGML_TYPE_F16) {- rope_neox_cuda(- (const half *)src0_dd, (half *)dst_dd, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,- attn_factor, corr_dims, main_stream- );- } else {- GGML_ASSERT(false);- }- } else {- if (src0->type == GGML_TYPE_F32) {- rope_cuda(- (const float *)src0_dd, (float *)dst_dd, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,- attn_factor, corr_dims, main_stream- );- } else if (src0->type == GGML_TYPE_F16) {- rope_cuda(- (const half *)src0_dd, (half *)dst_dd, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,- attn_factor, corr_dims, main_stream- );- } else {+ switch (src0->type) {+ case GGML_TYPE_F16:+ get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_F32:+ get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q4_0:+ get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q4_1:+ get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q5_0:+ get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q5_1:+ get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q8_0:+ get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ default:+ // TODO: k-quants+ fprintf(stderr, "%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));GGML_ASSERT(false);- }+ break;}-- (void) src1;- (void) dst;- (void) src1_dd;}-static void ggml_cuda_op_alibi(+template<class op>+static void ggml_cuda_op_bin_bcast(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+ GGML_UNUSED(ctx);- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t nrows = ggml_nrows(src0);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);- //const int n_past = ((int32_t *) dst->op_params)[0];- const int n_head = ((int32_t *) dst->op_params)[1];- float max_bias;- memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));+ if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {+ op()(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {+ op()(src0, src1, dst, (const half *) src0_dd, src1_dd, (half *) dst_dd, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {+ op()(src0, src1, dst, (const half *) src0_dd, src1_dd, dst_dd, main_stream);+ } else {+ fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,+ ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));+ GGML_ASSERT(false);+ }+}- //GGML_ASSERT(ne01 + n_past == ne00);- GGML_ASSERT(n_head == ne02);+static void ggml_cuda_op_repeat(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t main_stream) {- const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(ctx, dst, src0, dst, nullptr, src0_d, dst_d, main_stream);- const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);+ GGML_UNUSED(src1);+ GGML_UNUSED(src1_d);+}- alibi_f32_cuda(src0_dd, dst_dd, ne00, nrows, ne01, n_heads_log2_floor, m0, m1, main_stream);+static void ggml_cuda_op_add(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {- (void) src1;- (void) src1_dd;+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(ctx, src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);}-static void ggml_cuda_op_pool2d(+static void ggml_cuda_op_acc(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->ne[3] == 1); // just 3D tensors supported- const int32_t * opts = (const int32_t *)dst->op_params;- enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);- const int k0 = opts[1];- const int k1 = opts[2];- const int s0 = opts[3];- const int s1 = opts[4];- const int p0 = opts[5];- const int p1 = opts[6];-- const int64_t IH = src0->ne[1];- const int64_t IW = src0->ne[0];-- const int64_t N = dst->ne[3];- const int64_t OC = dst->ne[2];- const int64_t OH = dst->ne[1];- const int64_t OW = dst->ne[0];+ int nb1 = dst->op_params[0] / 4; // 4 bytes of float32+ int nb2 = dst->op_params[1] / 4; // 4 bytes of float32+ // int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused+ int offset = dst->op_params[3] / 4; // offset in bytes- const int parallel_elements = N * OC * OH * OW;- const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE;- dim3 block_nums(num_blocks);- pool2d_nchw_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, main_stream>>>(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_dd, dst_dd, op);+ acc_f32_cuda(src0_dd, src1_dd, dst_dd, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, main_stream);- (void) src1;- (void) src1_dd;+ GGML_UNUSED(dst);}-static void ggml_cuda_op_im2col(+static void ggml_cuda_op_mul(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);-- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];- const int32_t s1 = ((const int32_t*)(dst->op_params))[1];- const int32_t p0 = ((const int32_t*)(dst->op_params))[2];- const int32_t p1 = ((const int32_t*)(dst->op_params))[3];- const int32_t d0 = ((const int32_t*)(dst->op_params))[4];- const int32_t d1 = ((const int32_t*)(dst->op_params))[5];-- const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;-- const int64_t IC = src1->ne[is_2D ? 2 : 1];- const int64_t IH = is_2D ? src1->ne[1] : 1;- const int64_t IW = src1->ne[0];+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(ctx, src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+}- const int64_t KH = is_2D ? src0->ne[1] : 1;- const int64_t KW = src0->ne[0];+static void ggml_cuda_op_div(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {- const int64_t OH = is_2D ? dst->ne[2] : 1;- const int64_t OW = dst->ne[1];+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(ctx, src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);+}- const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32- const int64_t batch = src1->ne[3];- const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32+static void ggml_cuda_op_gelu(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- if(dst->type == GGML_TYPE_F16) {- im2col_cuda(src1_dd, (half*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream);- } else {- im2col_cuda(src1_dd, (float*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream);- }+ gelu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- (void) src0;- (void) src0_dd;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_op_sum_rows(+static void ggml_cuda_op_silu(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-+ GGML_UNUSED(ctx);GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);- const int64_t ncols = src0->ne[0];- const int64_t nrows = ggml_nrows(src0);-- sum_rows_f32_cuda(src0_dd, dst_dd, ncols, nrows, main_stream);+ silu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- (void) src1;- (void) dst;- (void) src1_dd;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_op_argsort(+static void ggml_cuda_op_gelu_quick(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-+ GGML_UNUSED(ctx);GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_I32);-- const int64_t ncols = src0->ne[0];- const int64_t nrows = ggml_nrows(src0);-- enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- argsort_f32_i32_cuda(src0_dd, (int *)dst_dd, ncols, nrows, order, main_stream);+ gelu_quick_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- (void) src1;- (void) dst;- (void) src1_dd;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_op_diag_mask_inf(+static void ggml_cuda_op_tanh(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-+ GGML_UNUSED(ctx);GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int nrows0 = ggml_nrows(src0);-- const int n_past = ((int32_t *) dst->op_params)[0];-- diag_mask_inf_f32_cuda(src0_dd, dst_dd, ne00, nrows0, ne01, n_past, main_stream);+ tanh_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- (void) src1;- (void) dst;- (void) src1_dd;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_op_soft_max(+static void ggml_cuda_op_relu(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-+ GGML_UNUSED(ctx);GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);- GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional-- const int64_t ne00 = src0->ne[0];- const int64_t nrows_x = ggml_nrows(src0);- const int64_t nrows_y = src0->ne[1];-- float scale = 1.0f;- float max_bias = 0.0f;-- memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));- memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));-- // positions tensor- float * src2_dd = nullptr;- cuda_pool_alloc<float> src2_f;-- ggml_tensor * src2 = dst->src[2];- const bool use_src2 = src2 != nullptr;-- if (use_src2) {- const bool src2_on_device = src2->backend == GGML_BACKEND_TYPE_GPU;-- if (src2_on_device) {- ggml_tensor_extra_gpu * src2_extra = (ggml_tensor_extra_gpu *) src2->extra;- src2_dd = (float *) src2_extra->data_device[g_main_device];- } else {- src2_dd = src2_f.alloc(ggml_nelements(src2));- CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src2_dd, src2, 0, 0, 0, 1, main_stream));- }- }+ relu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, src2_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_op_scale(+static void ggml_cuda_op_hardsigmoid(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-+ GGML_UNUSED(ctx);GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);- float scale;- memcpy(&scale, dst->op_params, sizeof(float));-- scale_f32_cuda(src0_dd, dst_dd, scale, ggml_nelements(src0), main_stream);- CUDA_CHECK(cudaGetLastError());+ hardsigmoid_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- (void) src1;- (void) dst;- (void) src1_dd;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_op_clamp(+static void ggml_cuda_op_hardswish(+ ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {-+ GGML_UNUSED(ctx);GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);- float min;- float max;- memcpy(&min, dst->op_params, sizeof(float));- memcpy(&max, (float *) dst->op_params + 1, sizeof(float));-- clamp_f32_cuda(src0_dd, dst_dd, min, max, ggml_nelements(src0), main_stream);- CUDA_CHECK(cudaGetLastError());+ hardswish_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- (void) src1;- (void) dst;- (void) src1_dd;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_op_flatten(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const ggml_cuda_op_flatten_t op) {- const int64_t nrows0 = ggml_nrows(src0);-- const bool use_src1 = src1 != nullptr;- const int64_t nrows1 = use_src1 ? ggml_nrows(src1) : 1;-- GGML_ASSERT(!use_src1 || src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT);- GGML_ASSERT( dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT);-- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- ggml_tensor_extra_gpu * src1_extra = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr;- ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;-- const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT;- const bool src1_on_device = use_src1 && src1->backend == GGML_BACKEND_TYPE_GPU;- const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU;-- // dd = data device- float * src0_ddf = nullptr;- float * src1_ddf = nullptr;- float * dst_ddf = nullptr;-- cuda_pool_alloc<float> src0_f;- cuda_pool_alloc<float> src1_f;- cuda_pool_alloc<float> dst_f;-- ggml_cuda_set_device(g_main_device);- cudaStream_t main_stream = g_cudaStreams[g_main_device][0];-- if (src0_on_device) {- src0_ddf = (float *) src0_extra->data_device[g_main_device];- } else {- src0_ddf = src0_f.alloc(ggml_nelements(src0));- CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_ddf, src0, 0, 0, 0, nrows0, main_stream));- }-- if (use_src1) {- if (src1_on_device) {- src1_ddf = (float *) src1_extra->data_device[g_main_device];- } else {- src1_ddf = src1_f.alloc(ggml_nelements(src1));- CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src1_ddf, src1, 0, 0, 0, nrows1, main_stream));- }- }- if (dst_on_device) {- dst_ddf = (float *) dst_extra->data_device[g_main_device];- } else {- dst_ddf = dst_f.alloc(ggml_nelements(dst));- }+static void ggml_cuda_op_leaky_relu(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- // do the computation- op(src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, main_stream);- CUDA_CHECK(cudaGetLastError());+ float negative_slope;+ memcpy(&negative_slope, dst->op_params, sizeof(float));- // copy dst to host if necessary- if (!dst_on_device) {- CUDA_CHECK(cudaMemcpyAsync(dst->data, dst_ddf, ggml_nbytes(dst), cudaMemcpyDeviceToHost, main_stream));- }+ leaky_relu_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), negative_slope, main_stream);- if (dst->backend == GGML_BACKEND_TYPE_CPU) {- CUDA_CHECK(cudaDeviceSynchronize());- }+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-static void ggml_cuda_set_peer_access(const int n_tokens) {- static bool peer_access_enabled = false;-- const bool enable_peer_access = n_tokens <= GGML_CUDA_PEER_MAX_BATCH_SIZE;-- if (peer_access_enabled == enable_peer_access) {- return;- }--#ifdef NDEBUG- for (int id = 0; id < g_device_count; ++id) {- ggml_cuda_set_device(id);- CUDA_CHECK(cudaDeviceSynchronize());- }-- for (int id = 0; id < g_device_count; ++id) {- ggml_cuda_set_device(id);-- for (int id_other = 0; id_other < g_device_count; ++id_other) {- if (id == id_other) {- continue;- }- if (id != g_main_device && id_other != g_main_device) {- continue;- }+static void ggml_cuda_op_sqr(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- int can_access_peer;- CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, id_other));- if (can_access_peer) {- if (enable_peer_access) {- cudaError_t err = cudaDeviceEnablePeerAccess(id_other, 0);- if (err != cudaErrorPeerAccessAlreadyEnabled) {- CUDA_CHECK(err);- }- } else {- cudaError_t err = cudaDeviceDisablePeerAccess(id_other);- if (err != cudaErrorPeerAccessNotEnabled) {- CUDA_CHECK(err);- }- }- }- }- }-#endif // NDEBUG+ sqr_f32_cuda(src0_dd, dst_dd, ggml_nelements(src0), main_stream);- peer_access_enabled = enable_peer_access;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-// FIXME: move this somewhere else-struct ggml_backend_cuda_split_buffer_type_context {- std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;-};--static void ggml_cuda_op_mul_mat(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op,- const bool convert_src1_to_q8_1) {+static void ggml_cuda_op_norm(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t ne03 = src0->ne[3];-- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];- const int64_t ne13 = src1->ne[3];- const int64_t nrows1 = ggml_nrows(src1);-- GGML_ASSERT(ne03 == ne13);-- const int64_t ne0 = dst->ne[0];- const int64_t ne1 = dst->ne[1];-- const int nb2 = dst->nb[2];- const int nb3 = dst->nb[3];-- GGML_ASSERT(dst->backend != GGML_BACKEND_TYPE_GPU_SPLIT);- GGML_ASSERT(src1->backend != GGML_BACKEND_TYPE_GPU_SPLIT);- GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1));+ const int64_t nrows = ggml_nrows(src0);- GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0);+ float eps;+ memcpy(&eps, dst->op_params, sizeof(float));- const int64_t i02_divisor = ne12 / ne02;+ norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, eps, main_stream);- const size_t src0_ts = ggml_type_size(src0->type);- const size_t src0_bs = ggml_blck_size(src0->type);- const size_t q8_1_ts = sizeof(block_q8_1);- const size_t q8_1_bs = QK8_1;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;- ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;+static void ggml_cuda_op_group_norm(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- const bool src0_on_device = src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT;- const bool src0_is_contiguous = ggml_is_contiguous(src0);- const bool src1_is_contiguous = ggml_is_contiguous(src1);+ int num_groups = dst->op_params[0];+ int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);+ group_norm_f32_cuda(src0_dd, dst_dd, num_groups * src0->ne[3], group_size, ggml_nelements(src0), main_stream);- const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT;- GGML_ASSERT(!(split && ne02 > 1));- GGML_ASSERT(!(split && ne03 > 1));- GGML_ASSERT(!(split && ne02 < ne12));+static void ggml_cuda_op_concat(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);- std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;- if (split) {- // TODO: check that src0->buffer->buft is a split buffer type, replace GGML_BACKEND_TYPE_GPU_SPLIT check- // GGML_ASSERT(src0->buffer != nullptr && src0->buffer->buft == ...);- ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;- tensor_split = buft_ctx->tensor_split;+ for (int i3 = 0; i3 < dst->ne[3]; i3++) {+ concat_f32_cuda(src0_dd + i3 * (src0->nb[3] / 4), src1_dd + i3 * (src1->nb[3] / 4), dst_dd + i3 * (dst->nb[3] / 4), dst->ne[0], dst->ne[1], dst->ne[2], src0->ne[2], main_stream);}- struct dev_data {- cuda_pool_alloc<char> src0_dd_alloc;- cuda_pool_alloc<float> src1_ddf_alloc;- cuda_pool_alloc<char> src1_ddq_alloc;- cuda_pool_alloc<float> dst_dd_alloc;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+}- char * src0_dd = nullptr;- float * src1_ddf = nullptr; // float- char * src1_ddq = nullptr; // q8_1- float * dst_dd = nullptr;+static void ggml_cuda_op_upscale(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);+ GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors- int64_t row_low;- int64_t row_high;- };+ const int scale_factor = dst->op_params[0];- dev_data dev[GGML_CUDA_MAX_DEVICES];+ upscale_f32_cuda(src0_dd, dst_dd, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], scale_factor, main_stream);- int used_devices = 0;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- for (int id = 0; id < g_device_count; ++id) {- // by default, use all rows- dev[id].row_low = 0;- dev[id].row_high = ne01;+static void ggml_cuda_op_pad(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);+ GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors++ pad_f32_cuda(src0_dd, dst_dd,+ src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],+ dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], main_stream);- // for multi GPU, get the row boundaries from tensor split- // and round to mul_mat_q tile sizes- if (split) {- const int64_t rounding = get_row_rounding(src0->type, tensor_split);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- if (id != 0) {- dev[id].row_low = ne01*tensor_split[id];- if (dev[id].row_low < ne01) {- dev[id].row_low -= dev[id].row_low % rounding;- }- }+static void ggml_cuda_op_arange(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);- if (id != g_device_count - 1) {- dev[id].row_high = ne01*tensor_split[id + 1];- if (dev[id].row_high < ne01) {- dev[id].row_high -= dev[id].row_high % rounding;- }- }- }- }+ float start;+ float stop;+ float step;+ memcpy(&start, (float *)dst->op_params + 0, sizeof(float));+ memcpy(&stop, (float *)dst->op_params + 1, sizeof(float));+ memcpy(&step, (float *)dst->op_params + 2, sizeof(float));- for (int id = 0; id < g_device_count; ++id) {- if ((!split && id != g_main_device) || dev[id].row_low == dev[id].row_high) {- continue;- }+ int64_t steps = (int64_t)ceil((stop - start) / step);+ GGML_ASSERT(ggml_nelements(dst) == steps);- used_devices++;+ arange_f32_cuda(dst_dd, dst->ne[0], start, step, main_stream);- const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device;- const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device;+ GGML_UNUSED(src0);+ GGML_UNUSED(src1);+ GGML_UNUSED(src0_dd);+ GGML_UNUSED(src1_dd);+}- ggml_cuda_set_device(id);- cudaStream_t stream = g_cudaStreams[id][0];+static void ggml_cuda_op_timestep_embedding(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);- if (src0_on_device && src0_is_contiguous) {- dev[id].src0_dd = (char *) src0_extra->data_device[id];- } else {- dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ggml_nbytes(src0));- }+ const int dim = dst->op_params[0];+ const int max_period = dst->op_params[1];- if (src1_on_device && src1_is_contiguous) {- dev[id].src1_ddf = (float *) src1_extra->data_device[id];- } else {- dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ggml_nelements(src1));- }+ timestep_embedding_f32_cuda(src0_dd, dst_dd, src0->ne[0], dst->nb[1], dim, max_period, main_stream);- if (convert_src1_to_q8_1) {- dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- if (src1_on_device && src1_is_contiguous) {- quantize_row_q8_1_cuda(dev[id].src1_ddf, dev[id].src1_ddq, ne10, nrows1, src1_padded_col_size, stream);- CUDA_CHECK(cudaGetLastError());- }- }+static void ggml_cuda_op_rms_norm(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- if (dst_on_device) {- dev[id].dst_dd = (float *) dst_extra->data_device[id];- } else {- const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst);- dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(size_dst_ddf);- }- }+ const int64_t ne00 = src0->ne[0];+ const int64_t nrows = ggml_nrows(src0);- // if multiple devices are used they need to wait for the main device- // here an event is recorded that signals that the main device has finished calculating the input data- if (split && used_devices > 1) {- ggml_cuda_set_device(g_main_device);- CUDA_CHECK(cudaEventRecord(src0_extra->events[g_main_device][0], g_cudaStreams[g_main_device][0]));- }+ float eps;+ memcpy(&eps, dst->op_params, sizeof(float));- const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11;- for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) {- const int64_t is = split ? (src1_col_0/src1_col_stride) % MAX_STREAMS : 0;- const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride;+ rms_norm_f32_cuda(src0_dd, dst_dd, ne00, nrows, eps, main_stream);- for (int id = 0; id < g_device_count; ++id) {- if ((!split && id != g_main_device) || dev[id].row_low == dev[id].row_high) {- continue;- }+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- const bool src1_on_device = src1->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device;- const bool dst_on_device = dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device;- const int64_t row_diff = dev[id].row_high - dev[id].row_low;+static void ggml_cuda_op_mul_mat_q(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream) {- ggml_cuda_set_device(id);- cudaStream_t stream = g_cudaStreams[id][is];+ const int64_t ne00 = src0->ne[0];- // wait for main GPU data if necessary- if (split && (id != g_main_device || is != 0)) {- CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[g_main_device][0], 0));- }+ const int64_t ne10 = src1->ne[0];+ GGML_ASSERT(ne10 % QK8_1 == 0);- for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) {- const int64_t i03 = i0 / ne12;- const int64_t i02 = i0 % ne12;+ const int64_t ne0 = dst->ne[0];- const size_t src1_ddq_i_offset = (i0*ne11 + src1_col_0) * src1_padded_col_size*q8_1_ts/q8_1_bs;+ const int64_t row_diff = row_high - row_low;- // for split tensors the data begins at i0 == i0_offset_low- char * src0_dd_i = dev[id].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;- float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10;- char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset;- float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);+ int id = ggml_cuda_get_device();- // the main device memory buffer can be on VRAM scratch, with space for all partial results- // in that case an offset on dst_ddf_i is needed- if (dst->backend == GGML_BACKEND_TYPE_GPU && id == g_main_device) {- dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split- }+ // the main device has a larger memory buffer to hold the results from all GPUs+ // nrows_dst == nrows of the matrix that the kernel writes into+ const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;- // copy src0, src1 to device if necessary- if (src1->backend == GGML_BACKEND_TYPE_GPU && src1_is_contiguous) {- if (id != g_main_device) {- if (convert_src1_to_q8_1) {- char * src1_ddq_i_source = dev[g_main_device].src1_ddq + src1_ddq_i_offset;- CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddq_i, id, src1_ddq_i_source, g_main_device,- src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream));- } else {- float * src1_ddf_i_source = (float *) src1_extra->data_device[g_main_device];- src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10;- CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, g_main_device,- src1_ncols*ne10*sizeof(float), stream));- }- }- } else if (src1->backend == GGML_BACKEND_TYPE_CPU || (src1_on_device && !src1_is_contiguous)) {- CUDA_CHECK(ggml_cuda_cpy_tensor_2d(- src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream));- } else {- GGML_ASSERT(false);- }+ switch (src0->type) {+ case GGML_TYPE_Q4_0:+ ggml_mul_mat_q4_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q4_1:+ ggml_mul_mat_q4_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q5_0:+ ggml_mul_mat_q5_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q5_1:+ ggml_mul_mat_q5_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q8_0:+ ggml_mul_mat_q8_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q2_K:+ ggml_mul_mat_q2_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q3_K:+ ggml_mul_mat_q3_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q4_K:+ ggml_mul_mat_q4_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q5_K:+ ggml_mul_mat_q5_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ case GGML_TYPE_Q6_K:+ ggml_mul_mat_q6_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ break;+ default:+ GGML_ASSERT(false);+ break;+ }- if (convert_src1_to_q8_1 && (src1->backend == GGML_BACKEND_TYPE_CPU || !src1_is_contiguous)) {- quantize_row_q8_1_cuda(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, src1_padded_col_size, stream);- CUDA_CHECK(cudaGetLastError());- }+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_ddf_i);+}- if (src1_col_0 == 0 && (!src0_on_device || !src0_is_contiguous) && i02 % i02_divisor == 0) {- CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream));- }+static void ggml_cuda_op_mul_mat_vec_q(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream) {- // do the computation- op(src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i,- dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream);- CUDA_CHECK(cudaGetLastError());+ const int64_t ne00 = src0->ne[0];+ const int64_t row_diff = row_high - row_low;- // copy dst to host or other device if necessary- if (!dst_on_device) {- void * dst_off_device;- cudaMemcpyKind kind;- if (dst->backend == GGML_BACKEND_TYPE_CPU) {- dst_off_device = dst->data;- kind = cudaMemcpyDeviceToHost;- } else if (dst->backend == GGML_BACKEND_TYPE_GPU) {- dst_off_device = dst_extra->data_device[g_main_device];- kind = cudaMemcpyDeviceToDevice;- } else {- GGML_ASSERT(false);- }- if (split) {- // src0 = weight matrix is saved as a transposed matrix for better memory layout.- // dst is NOT transposed.- // The outputs of matrix matrix multiplications can therefore NOT simply be concatenated for >1 GPU.- // Instead they need to be copied to the correct slice in ne0 = dst row index.- // If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results.- float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);- GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));- dhf_dst_i += src1_col_0*ne0 + dev[id].row_low;-#if !defined(GGML_USE_HIPBLAS)- if (kind == cudaMemcpyDeviceToDevice) {- // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices- cudaMemcpy3DPeerParms p = {};- p.dstDevice = g_main_device;- p.dstPtr = make_cudaPitchedPtr(dhf_dst_i, ne0*sizeof(float), row_diff, src1_ncols);- p.srcDevice = id;- p.srcPtr = make_cudaPitchedPtr(dst_dd_i, row_diff*sizeof(float), row_diff, src1_ncols);- p.extent = make_cudaExtent(row_diff*sizeof(float), src1_ncols, 1);- CUDA_CHECK(cudaMemcpy3DPeerAsync(&p, stream));- } else-#endif- {- CUDA_CHECK(cudaMemcpy2DAsync(dhf_dst_i, ne0*sizeof(float),- dst_dd_i, row_diff*sizeof(float),- row_diff*sizeof(float), src1_ncols,- kind, stream));- }- } else {- float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);- GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));- dhf_dst_i += src1_col_0*ne0;- CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), kind, stream));- }- }+ const int64_t ne10 = src1->ne[0];+ GGML_ASSERT(ne10 % QK8_1 == 0);- // add event for the main device to wait on until other device is done- if (split && (id != g_main_device || is != 0)) {- CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream));- }- }- }- }+ const int64_t ne0 = dst->ne[0];- // main device waits for all other devices to be finished- if (split && g_device_count > 1) {- int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE;- is_max = is_max <= MAX_STREAMS ? is_max : MAX_STREAMS;+ int id;+ CUDA_CHECK(cudaGetDevice(&id));- ggml_cuda_set_device(g_main_device);- for (int id = 0; id < g_device_count; ++id) {- if (dev[id].row_low == dev[id].row_high) {- continue;- }- for (int64_t is = 0; is < is_max; ++is) {- CUDA_CHECK(cudaStreamWaitEvent(g_cudaStreams[g_main_device][0], src0_extra->events[id][is], 0));- }- }- }+ // the main device has a larger memory buffer to hold the results from all GPUs+ // nrows_dst == nrows of the matrix that the kernel writes into+ const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;- if (dst->backend == GGML_BACKEND_TYPE_CPU) {- ggml_cuda_set_device(g_main_device);- CUDA_CHECK(cudaDeviceSynchronize());+ switch (src0->type) {+ case GGML_TYPE_Q4_0:+ mul_mat_vec_q_cuda<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q4_1:+ mul_mat_vec_q_cuda<QK4_1, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q5_0:+ mul_mat_vec_q_cuda<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q5_1:+ mul_mat_vec_q_cuda<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q8_0:+ mul_mat_vec_q_cuda<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q2_K:+ mul_mat_vec_q_cuda<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q3_K:+ mul_mat_vec_q_cuda<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q4_K:+ mul_mat_vec_q_cuda<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q5_K:+ mul_mat_vec_q_cuda<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_Q6_K:+ mul_mat_vec_q_cuda<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ2_XXS:+ mul_mat_vec_q_cuda<QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ2_XS:+ mul_mat_vec_q_cuda<QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ2_S:+ mul_mat_vec_q_cuda<QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ3_XXS:+ mul_mat_vec_q_cuda<QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ1_S:+ mul_mat_vec_q_cuda<QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ4_NL:+ mul_mat_vec_q_cuda<QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ4_XS:+ mul_mat_vec_q_cuda<QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ case GGML_TYPE_IQ3_S:+ mul_mat_vec_q_cuda<QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_padded_row_size, src1_ncols, nrows_dst, stream);+ break;+ default:+ GGML_ASSERT(false);+ break;}-}--static void ggml_cuda_repeat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_repeat);-}--static void ggml_cuda_get_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_get_rows);-}--static void ggml_cuda_add(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_add);-}--static void ggml_cuda_acc(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_acc);-}-static void ggml_cuda_mul(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_mul);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_ddf_i);+ GGML_UNUSED(src1_ncols);+ GGML_UNUSED(src1_padded_row_size);}-static void ggml_cuda_div(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_div);-}--static void ggml_cuda_gelu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_gelu);-}--static void ggml_cuda_silu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_silu);-}--static void ggml_cuda_gelu_quick(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_gelu_quick);-}--static void ggml_cuda_tanh(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_tanh);-}--static void ggml_cuda_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_relu);-}--static void ggml_cuda_hardsigmoid(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardsigmoid);-}--static void ggml_cuda_hardswish(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_hardswish);-}-static void ggml_cuda_leaky_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_leaky_relu);-}+static void ggml_cuda_op_dequantize_mul_mat_vec(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream) {+ GGML_UNUSED(ctx);+ const int64_t ne00 = src0->ne[0];+ const int64_t row_diff = row_high - row_low;-static void ggml_cuda_sqr(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_sqr);-}+ GGML_ASSERT(src1->type == GGML_TYPE_F32);-static void ggml_cuda_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_norm);-}+ // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics+#ifdef GGML_CUDA_F16+ ggml_cuda_pool_alloc<half> src1_dfloat_a(ctx.pool());+ half * src1_dfloat = nullptr; // dfloat == half-static void ggml_cuda_group_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_group_norm);-}+ bool src1_convert_f16 =+ src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||+ src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||+ src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;-static void ggml_cuda_concat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_concat);-}+ if (src1_convert_f16) {+ src1_dfloat = src1_dfloat_a.alloc(ne00);+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ GGML_ASSERT(to_fp16_cuda != nullptr);+ to_fp16_cuda(src1_ddf_i, src1_dfloat, ne00, stream);+ }+#else+ const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion+#endif // GGML_CUDA_F16-static void ggml_cuda_upscale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_upscale);-}+ switch (src0->type) {+ case GGML_TYPE_Q4_0:+ dequantize_mul_mat_vec_q4_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q4_1:+ dequantize_mul_mat_vec_q4_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q5_0:+ dequantize_mul_mat_vec_q5_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q5_1:+ dequantize_mul_mat_vec_q5_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q8_0:+ dequantize_mul_mat_vec_q8_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q2_K:+ dequantize_mul_mat_vec_q2_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q3_K:+ dequantize_mul_mat_vec_q3_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q4_K:+ dequantize_mul_mat_vec_q4_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q5_K:+ dequantize_mul_mat_vec_q5_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_Q6_K:+ dequantize_mul_mat_vec_q6_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;+ case GGML_TYPE_F16:+ convert_mul_mat_vec_f16_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;+ default:+ GGML_ASSERT(false);+ break;+ }-static void ggml_cuda_pad(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_pad);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_ddq_i);+ GGML_UNUSED(src1_ncols);+ GGML_UNUSED(src1_padded_row_size);}-static void ggml_cuda_rms_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_rms_norm);-}+static void ggml_cuda_op_mul_mat_cublas(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream) {-GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {- if (!g_cublas_loaded) return false;+ GGML_ASSERT(src0_dd_i != nullptr);+ GGML_ASSERT(src1_ddf_i != nullptr);+ GGML_ASSERT(dst_dd_i != nullptr);+ const int64_t ne00 = src0->ne[0];const int64_t ne10 = src1->ne[0];const int64_t ne0 = dst->ne[0];- const int64_t ne1 = dst->ne[1];-- // TODO: find the optimal values for these- return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) &&- src1->type == GGML_TYPE_F32 &&- dst->type == GGML_TYPE_F32 &&- (ne0 >= 32 && ne1 >= 32 && ne10 >= 32);-}--static void ggml_cuda_mul_mat_vec_p021(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){- GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1));- GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT);- GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation- GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // 0213 permutation- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);-- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];-- const int64_t ne12 = src1->ne[2];-- ggml_cuda_set_device(g_main_device);- cudaStream_t main_stream = g_cudaStreams[g_main_device][0];-- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- void * src0_ddq = src0_extra->data_device[g_main_device];-- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;- float * src1_ddf = (float *) src1_extra->data_device[g_main_device];-- ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;- float * dst_ddf = (float *) dst_extra->data_device[g_main_device];- ggml_mul_mat_p021_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, ne02, ne12, main_stream);-}+ const int64_t row_diff = row_high - row_low;-static void ggml_cuda_mul_mat_vec_nc(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){- GGML_ASSERT(!ggml_is_transposed(src0));- GGML_ASSERT(!ggml_is_transposed(src1));- GGML_ASSERT(!ggml_is_permuted(src0));- GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT);- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);+ int id = ggml_cuda_get_device();- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];+ // the main device has a larger memory buffer to hold the results from all GPUs+ // ldc == nrows of the matrix that cuBLAS writes into+ int ldc = id == ctx.device ? ne0 : row_diff;- const int64_t nb01 = src0->nb[1];- const int64_t nb02 = src0->nb[2];+ const int compute_capability = get_cuda_global_info().devices[id].cc;- const int64_t ne12 = src1->ne[2];+ if (compute_capability >= CC_VOLTA && (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) {+ // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32+ ggml_cuda_pool_alloc<half> src0_as_f16(ctx.pool());+ if (src0->type != GGML_TYPE_F16) {+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type);+ GGML_ASSERT(to_fp16_cuda != nullptr);+ size_t ne = row_diff*ne00;+ src0_as_f16.alloc(ne);+ to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream);+ }+ const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get();- ggml_cuda_set_device(g_main_device);- cudaStream_t main_stream = g_cudaStreams[g_main_device][0];+ ggml_cuda_pool_alloc<half> src1_as_f16(ctx.pool());+ if (src1->type != GGML_TYPE_F16) {+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ GGML_ASSERT(to_fp16_cuda != nullptr);+ size_t ne = src1_ncols*ne10;+ src1_as_f16.alloc(ne);+ to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream);+ }+ const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get();+ ggml_cuda_pool_alloc<half> dst_f16(ctx.pool(), row_diff*src1_ncols);- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- void * src0_ddq = src0_extra->data_device[g_main_device];+ const half alpha_f16 = 1.0f;+ const half beta_f16 = 0.0f;- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;- float * src1_ddf = (float *) src1_extra->data_device[g_main_device];+ CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));+ CUBLAS_CHECK(+ cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,+ row_diff, src1_ncols, ne10,+ &alpha_f16, src0_ptr, CUDA_R_16F, ne00,+ src1_ptr, CUDA_R_16F, ne10,+ &beta_f16, dst_f16.get(), CUDA_R_16F, ldc,+ CUBLAS_COMPUTE_16F,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));- ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;- float * dst_ddf = (float *) dst_extra->data_device[g_main_device];+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);+ to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);+ } else {+ ggml_cuda_pool_alloc<float> src0_ddq_as_f32(ctx.pool(id));+ ggml_cuda_pool_alloc<float> src1_ddq_as_f32(ctx.pool(id));- const int64_t row_stride_x = nb01 / sizeof(half);- const int64_t channel_stride_x = nb02 / sizeof(half);+ if (src0->type != GGML_TYPE_F32) {+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type);+ GGML_ASSERT(to_fp32_cuda != nullptr);+ src0_ddq_as_f32.alloc(row_diff*ne00);+ to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream);+ }+ if (src1->type != GGML_TYPE_F32) {+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type);+ GGML_ASSERT(to_fp32_cuda != nullptr);+ src1_ddq_as_f32.alloc(src1_ncols*ne10);+ to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream);+ }- ggml_mul_mat_vec_nc_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, row_stride_x, ne02, ne12, channel_stride_x, main_stream);-}+ const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get();+ const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();-static __global__ void k_compute_batched_ptrs(- const half * src0_as_f16, const half * src1_as_f16, char * dst,- const void ** ptrs_src, void ** ptrs_dst,- int64_t ne12, int64_t ne13,- int64_t ne23,- size_t nb02, size_t nb03,- size_t nb12, size_t nb13,- size_t nbd2, size_t nbd3,- int64_t r2, int64_t r3) {- int64_t i13 = blockIdx.x * blockDim.x + threadIdx.x;- int64_t i12 = blockIdx.y * blockDim.y + threadIdx.y;+ const float alpha = 1.0f;+ const float beta = 0.0f;- if (i13 >= ne13 || i12 >= ne12) {- return;+ CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));+ CUBLAS_CHECK(+ cublasSgemm(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,+ row_diff, src1_ncols, ne10,+ &alpha, src0_ddf_i, ne00,+ src1_ddf1_i, ne10,+ &beta, dst_dd_i, ldc));}- int64_t i03 = i13 / r3;- int64_t i02 = i12 / r2;-- ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03;- ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12 + i13*nb13;- ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_ddq_i);+ GGML_UNUSED(src1_padded_row_size);}-static void ggml_cuda_mul_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- GGML_ASSERT(!ggml_is_transposed(src0));- GGML_ASSERT(!ggml_is_transposed(src1));-- GGML_ASSERT(src0->backend != GGML_BACKEND_TYPE_GPU_SPLIT);- GGML_ASSERT(src0->type == GGML_TYPE_F16);-- GGML_TENSOR_BINARY_OP_LOCALS+static void ggml_cuda_op_rope(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);+ GGML_ASSERT(src0->type == dst->type);- const int64_t ne_dst = ggml_nelements(dst);+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int64_t ne2 = dst->ne[2];+ const int64_t nrows = ggml_nrows(src0);- ggml_cuda_set_device(g_main_device);- cudaStream_t main_stream = g_cudaStreams[g_main_device][0];+ //const int n_past = ((int32_t *) dst->op_params)[0];+ const int n_dims = ((int32_t *) dst->op_params)[1];+ const int mode = ((int32_t *) dst->op_params)[2];+ const int n_ctx = ((int32_t *) dst->op_params)[3];+ const int n_orig_ctx = ((int32_t *) dst->op_params)[4];- CUBLAS_CHECK(cublasSetStream(g_cublas_handles[g_main_device], main_stream));+ // RoPE alteration for extended context+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- void * src0_ddq = src0_extra->data_device[g_main_device];- half * src0_f16 = (half *) src0_ddq;+ const int32_t * pos = nullptr;+ if ((mode & 1) == 0) {+ GGML_ASSERT(src1->type == GGML_TYPE_I32);+ GGML_ASSERT(src1->ne[0] == ne2);+ pos = (const int32_t *) src1_dd;+ }- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;- float * src1_ddf = (float *) src1_extra->data_device[g_main_device];+ const bool is_neox = mode & 2;+ const bool is_glm = mode & 4;- ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;- float * dst_ddf = (float *) dst_extra->data_device[g_main_device];+ rope_corr_dims corr_dims;+ ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims.v);- // convert src1 to fp16- cuda_pool_alloc<half> src1_f16_alloc;- if (src1->type != GGML_TYPE_F16) {- const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);- const int64_t ne_src1 = ggml_nelements(src1);- src1_f16_alloc.alloc(ne_src1);- GGML_ASSERT(to_fp16_cuda != nullptr);- to_fp16_cuda(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream);+ // compute+ if (is_glm) {+ GGML_ASSERT(false);+ rope_glm_f32_cuda(src0_dd, dst_dd, ne00, nrows, pos, freq_scale, ne01, freq_base, n_ctx, main_stream);+ } else if (is_neox) {+ if (src0->type == GGML_TYPE_F32) {+ rope_neox_cuda(+ (const float *)src0_dd, (float *)dst_dd, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,+ attn_factor, corr_dims, main_stream+ );+ } else if (src0->type == GGML_TYPE_F16) {+ rope_neox_cuda(+ (const half *)src0_dd, (half *)dst_dd, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,+ attn_factor, corr_dims, main_stream+ );+ } else {+ GGML_ASSERT(false);+ }+ } else {+ if (src0->type == GGML_TYPE_F32) {+ rope_cuda(+ (const float *)src0_dd, (float *)dst_dd, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,+ attn_factor, corr_dims, main_stream+ );+ } else if (src0->type == GGML_TYPE_F16) {+ rope_cuda(+ (const half *)src0_dd, (half *)dst_dd, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,+ attn_factor, corr_dims, main_stream+ );+ } else {+ GGML_ASSERT(false);+ }}- half * src1_f16 = src1->type == GGML_TYPE_F16 ? (half *) src1_ddf : src1_f16_alloc.get();- cuda_pool_alloc<half> dst_f16;- char * dst_t;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- cublasComputeType_t cu_compute_type = CUBLAS_COMPUTE_16F;- cudaDataType_t cu_data_type = CUDA_R_16F;+static void ggml_cuda_op_alibi(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- // dst strides- size_t nbd2 = dst->nb[2];- size_t nbd3 = dst->nb[3];+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int64_t ne02 = src0->ne[2];+ const int64_t nrows = ggml_nrows(src0);- const half alpha_f16 = 1.0f;- const half beta_f16 = 0.0f;+ //const int n_past = ((int32_t *) dst->op_params)[0];+ const int n_head = ((int32_t *) dst->op_params)[1];+ float max_bias;+ memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));- const float alpha_f32 = 1.0f;- const float beta_f32 = 0.0f;+ //GGML_ASSERT(ne01 + n_past == ne00);+ GGML_ASSERT(n_head == ne02);- const void * alpha = &alpha_f16;- const void * beta = &beta_f16;+ const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));- if (dst->op_params[0] == GGML_PREC_DEFAULT) {- dst_t = (char *) dst_f16.alloc(ne_dst);+ const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);- nbd2 /= sizeof(float) / sizeof(half);- nbd3 /= sizeof(float) / sizeof(half);- } else {- dst_t = (char *) dst_ddf;+ alibi_f32_cuda(src0_dd, dst_dd, ne00, nrows, ne01, n_heads_log2_floor, m0, m1, main_stream);- cu_compute_type = CUBLAS_COMPUTE_32F;- cu_data_type = CUDA_R_32F;+ GGML_UNUSED(src1);+ GGML_UNUSED(src1_dd);+}- alpha = &alpha_f32;- beta = &beta_f32;- }+static void ggml_cuda_op_pool2d(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- GGML_ASSERT(ne12 % ne02 == 0);- GGML_ASSERT(ne13 % ne03 == 0);+ const int32_t * opts = (const int32_t *)dst->op_params;+ enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);+ const int k0 = opts[1];+ const int k1 = opts[2];+ const int s0 = opts[3];+ const int s1 = opts[4];+ const int p0 = opts[5];+ const int p1 = opts[6];- // broadcast factors- const int64_t r2 = ne12/ne02;- const int64_t r3 = ne13/ne03;+ const int64_t IH = src0->ne[1];+ const int64_t IW = src0->ne[0];-#if 0- // use cublasGemmEx- {- for (int i13 = 0; i13 < ne13; ++i13) {- for (int i12 = 0; i12 < ne12; ++i12) {- int i03 = i13 / r3;- int i02 = i12 / r2;+ const int64_t N = dst->ne[3];+ const int64_t OC = dst->ne[2];+ const int64_t OH = dst->ne[1];+ const int64_t OW = dst->ne[0];- CUBLAS_CHECK(- cublasGemmEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,- ne01, ne11, ne10,- alpha, (const char *) src0_as_f16 + i02*src0->nb[2] + i03*src0->nb[3] , CUDA_R_16F, nb01/sizeof(half),- (const char *) src1_as_f16 + i12*src1->nb[2]/2 + i13*src1->nb[3]/2, CUDA_R_16F, nb11/sizeof(float),- beta, ( char *) dst_t + i12*nbd2 + i13*nbd3, cu_data_type, ne01,- cu_compute_type,- CUBLAS_GEMM_DEFAULT_TENSOR_OP));- }- }- }-#else- if (r2 == 1 && r3 == 1 && src0->nb[2]*src0->ne[2] == src0->nb[3] && src1->nb[2]*src1->ne[2] == src1->nb[3]) {- // there is no broadcast and src0, src1 are contiguous across dims 2, 3- // use cublasGemmStridedBatchedEx- CUBLAS_CHECK(- cublasGemmStridedBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,- ne01, ne11, ne10,- alpha, (const char *) src0_f16, CUDA_R_16F, nb01/nb00, nb02/nb00, // strideA- (const char *) src1_f16, CUDA_R_16F, nb11/nb10, nb12/nb10, // strideB- beta, ( char *) dst_t, cu_data_type, ne01, nb2/nb0, // strideC- ne12*ne13,- cu_compute_type,- CUBLAS_GEMM_DEFAULT_TENSOR_OP));- } else {- // use cublasGemmBatchedEx- const int ne23 = ne12*ne13;+ const int parallel_elements = N * OC * OH * OW;+ const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE;+ dim3 block_nums(num_blocks);+ pool2d_nchw_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, main_stream>>>(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_dd, dst_dd, op);- cuda_pool_alloc<const void *> ptrs_src(2*ne23);- cuda_pool_alloc< void *> ptrs_dst(1*ne23);+ GGML_UNUSED(src1);+ GGML_UNUSED(src1_dd);+}- dim3 block_dims(ne13, ne12);- k_compute_batched_ptrs<<<1, block_dims, 0, main_stream>>>(- src0_f16, src1_f16, dst_t,- ptrs_src.get(), ptrs_dst.get(),- ne12, ne13,- ne23,- nb02, nb03,- src1->type == GGML_TYPE_F16 ? nb12 : nb12/2,- src1->type == GGML_TYPE_F16 ? nb13 : nb13/2,- nbd2, nbd3,- r2, r3);- CUDA_CHECK(cudaGetLastError());+static void ggml_cuda_op_im2col(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);- CUBLAS_CHECK(- cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,- ne01, ne11, ne10,- alpha, (const void **) (ptrs_src.get() + 0*ne23), CUDA_R_16F, nb01/nb00,- (const void **) (ptrs_src.get() + 1*ne23), CUDA_R_16F, nb11/nb10,- beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne01,- ne23,- cu_compute_type,- CUBLAS_GEMM_DEFAULT_TENSOR_OP));- }-#endif+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t*)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t*)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t*)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t*)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t*)(dst->op_params))[5];- if (dst->op_params[0] == GGML_PREC_DEFAULT) {- const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);- to_fp32_cuda(dst_f16.get(), dst_ddf, ne_dst, main_stream);- }-}+ const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;-static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- const bool all_on_device =- (src0->backend == GGML_BACKEND_TYPE_GPU || src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT) &&- (src1->backend == GGML_BACKEND_TYPE_GPU) &&- ( dst->backend == GGML_BACKEND_TYPE_GPU);+ const int64_t IC = src1->ne[is_2D ? 2 : 1];+ const int64_t IH = is_2D ? src1->ne[1] : 1;+ const int64_t IW = src1->ne[0];- const bool split = src0->backend == GGML_BACKEND_TYPE_GPU_SPLIT;+ const int64_t KH = is_2D ? src0->ne[1] : 1;+ const int64_t KW = src0->ne[0];- int64_t min_compute_capability = INT_MAX;+ const int64_t OH = is_2D ? dst->ne[2] : 1;+ const int64_t OW = dst->ne[1];- bool any_pascal_with_slow_fp16 = false;- if (split) {- ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;- auto & tensor_split = buft_ctx->tensor_split;- for (int id = 0; id < g_device_count; ++id) {- // skip devices that are not going to do any work:- if (tensor_split[id] >= (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) {- continue;- }+ const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32+ const int64_t batch = src1->ne[3];+ const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32- if (min_compute_capability > g_device_caps[id].cc) {- min_compute_capability = g_device_caps[id].cc;- }- if (g_device_caps[id].cc == 610) {- any_pascal_with_slow_fp16 = true;- }- }+ if(dst->type == GGML_TYPE_F16) {+ im2col_cuda(src1_dd, (half*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream);} else {- min_compute_capability = g_device_caps[g_main_device].cc;- any_pascal_with_slow_fp16 = g_device_caps[g_main_device].cc == 610;+ im2col_cuda(src1_dd, (float*) dst_dd, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, main_stream);}- // check data types and tensor shapes for custom matrix multiplication kernels:- bool use_dequantize_mul_mat_vec = (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16)- && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32- && src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->ne[1] == 1;+ GGML_UNUSED(src0);+ GGML_UNUSED(src0_dd);+}++static void ggml_cuda_op_sum_rows(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- bool use_mul_mat_vec_q = ggml_is_quantized(src0->type)- && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32- && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;+ const int64_t ncols = src0->ne[0];+ const int64_t nrows = ggml_nrows(src0);- bool use_mul_mat_q = ggml_cuda_supports_mmq(src0->type)- && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;+ sum_rows_f32_cuda(src0_dd, dst_dd, ncols, nrows, main_stream);-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- const bool fp16_performance_good = min_compute_capability >= CC_RDNA1;+static void ggml_cuda_op_argsort(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_I32);-#ifdef CUDA_USE_TENSOR_CORES- use_mul_mat_q = use_mul_mat_q && min_compute_capability < CC_RDNA3;-#endif // CUDA_USE_TENSOR_CORES+ const int64_t ncols = src0->ne[0];+ const int64_t nrows = ggml_nrows(src0);-#else+ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];- // fp16 performance is good on Volta or newer and on P100 (compute capability 6.0)- const bool fp16_performance_good = min_compute_capability >= CC_PASCAL && !any_pascal_with_slow_fp16;+ argsort_f32_i32_cuda(src0_dd, (int *)dst_dd, ncols, nrows, order, main_stream);- // mmvq and mmq need the __dp4a instruction which on NVIDIA is only available for CC >= 6.1- use_mul_mat_vec_q = use_mul_mat_vec_q && min_compute_capability >= MIN_CC_DP4A;- use_mul_mat_q = use_mul_mat_q && min_compute_capability >= MIN_CC_DP4A;+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}-#ifdef CUDA_USE_TENSOR_CORES- // when tensor cores are available, use them for large batch size- // ref: https://github.com/ggerganov/llama.cpp/pull/3776- use_mul_mat_q = use_mul_mat_q && (!fp16_performance_good || src1->ne[1] <= MMQ_MAX_BATCH_SIZE);-#endif // CUDA_USE_TENSOR_CORES+static void ggml_cuda_op_diag_mask_inf(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int nrows0 = ggml_nrows(src0);- // if mmvq is available it's a better choice than dmmv:-#ifndef GGML_CUDA_FORCE_DMMV- use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q;-#endif // GGML_CUDA_FORCE_DMMV+ const int n_past = ((int32_t *) dst->op_params)[0];- // debug helpers- //printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);- //printf(" %8d %8d %8d %8d\n", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]);- //printf("src1: %8d %8d %8d %8d\n", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]);- //printf(" %8d %8d %8d %8d\n", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]);- //printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);- //printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);+ diag_mask_inf_f32_cuda(src0_dd, dst_dd, ne00, nrows0, ne01, n_past, main_stream);- if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {- // KQ single-batch- ggml_cuda_mul_mat_vec_p021(src0, src1, dst);- } else if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {- // KQV single-batch- ggml_cuda_mul_mat_vec_nc(src0, src1, dst);- } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {- // KQ + KQV multi-batch- ggml_cuda_mul_mat_batched_cublas(src0, src1, dst);- } else if (use_dequantize_mul_mat_vec) {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false);- } else if (use_mul_mat_vec_q) {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true);- } else if (use_mul_mat_q) {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true);- } else {- ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false);- }+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);}-#if 0-template<typename ... Srcs>-static __global__ void k_compute_batched_ptrs_id(- const void ** ptrs_src, void ** ptrs_dst,- int ne12, int ne13,- int ne23,- int nb02, int nb03,- int nb12, int nb13,- int nb2, int nb3,- int r2, int r3,- ggml_type src0_type, half * src0_as_f16, int64_t src0_ne,- const half * src1_f16, half * dst_f16,- const int32_t * ids, const int id,- Srcs... src0s) {+static void ggml_cuda_op_soft_max(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- int i = ids[id];+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional- half * src0_f16;- const void * srcs_ar[] = { (const half *) src0s... };- if (src0_type == GGML_TYPE_F16) {- src0_f16 = (half *) srcs_ar[i];- } else {- src0_f16 = src0_as_f16;- if (threadIdx.x == 0 && threadIdx.y == 0) {- const to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(src0_type);- to_fp16(srcs_ar[i], src0_f16, src0_ne, cudaStreamFireAndForget);- }- }+ const int64_t ne00 = src0->ne[0];+ const int64_t nrows_x = ggml_nrows(src0);+ const int64_t nrows_y = src0->ne[1];- int i13 = blockIdx.x * blockDim.x + threadIdx.x;- int i12 = blockIdx.y * blockDim.y + threadIdx.y;+ float scale = 1.0f;+ float max_bias = 0.0f;- if (i13 >= ne13 || i12 >= ne12) {- return;- }+ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));+ memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));- int i03 = i13 / r3;- int i02 = i12 / r2;+ // positions tensor+ float * src2_dd = nullptr;- ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_f16 + i02*nb02 + i03*nb03;- ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_f16 + i12*nb12/2 + i13*nb13/2;- ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst_f16 + i12* nb2/2 + i13* nb3/2;+ ggml_tensor * src2 = dst->src[2];+ const bool use_src2 = src2 != nullptr;++ if (use_src2) {+ src2_dd = (float *)src2->data;+ }++ soft_max_f32_cuda(src0_dd, src1 ? src1_dd : nullptr, src2_dd, dst_dd, ne00, nrows_x, nrows_y, scale, max_bias, main_stream);}-static void ggml_cuda_mul_mat_id_cublas(ggml_tensor * dst) {- const struct ggml_tensor * ids = dst->src[0];- const struct ggml_tensor * src1 = dst->src[1];- const struct ggml_tensor * src00 = dst->src[2];+static void ggml_cuda_op_scale(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- const int id = dst->op_params[0];+ float scale;+ memcpy(&scale, dst->op_params, sizeof(float));- GGML_ASSERT(!ggml_is_transposed(src00));- GGML_ASSERT(!ggml_is_transposed(src1));+ scale_f32_cuda(src0_dd, dst_dd, scale, ggml_nelements(src0), main_stream);+ CUDA_CHECK(cudaGetLastError());- GGML_ASSERT(src00->backend != GGML_BACKEND_TYPE_GPU_SPLIT);- GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- const int64_t ne00 = src00->ne[0]; GGML_UNUSED(ne00);- const int64_t ne01 = src00->ne[1];- const int64_t ne02 = src00->ne[2];- const int64_t ne03 = src00->ne[3];+static void ggml_cuda_op_clamp(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {+ GGML_UNUSED(ctx);+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- //const int64_t nb01 = src00->nb[1];- const int64_t nb02 = src00->nb[2]; GGML_UNUSED(nb02);- const int64_t nb03 = src00->nb[3]; GGML_UNUSED(nb03);+ float min;+ float max;+ memcpy(&min, dst->op_params, sizeof(float));+ memcpy(&max, (float *) dst->op_params + 1, sizeof(float));- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];- const int64_t ne13 = src1->ne[3];+ clamp_f32_cuda(src0_dd, dst_dd, min, max, ggml_nelements(src0), main_stream);+ CUDA_CHECK(cudaGetLastError());- //const int64_t nb11 = src1->nb[1];- const int64_t nb12 = src1->nb[2]; GGML_UNUSED(nb12);- const int64_t nb13 = src1->nb[3]; GGML_UNUSED(nb13);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_dd);+}- const int64_t ne1 = ggml_nelements(src1);- const int64_t ne = ggml_nelements(dst);+// TODO: remove this function+static void ggml_cuda_op_flatten(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const ggml_cuda_op_flatten_t op) {+ GGML_ASSERT(!src0 || ggml_backend_buffer_is_cuda(src0->buffer));+ GGML_ASSERT(!src1 || ggml_backend_buffer_is_cuda(src1->buffer));+ GGML_ASSERT( ggml_backend_buffer_is_cuda(dst->buffer));- ggml_cuda_set_device(g_main_device);- cudaStream_t main_stream = g_cudaStreams[g_main_device][0];+ // dd = data device+ float * src0_ddf = src0 ? (float *) src0->data : nullptr;+ float * src1_ddf = src1 ? (float *) src1->data : nullptr;+ float * dst_ddf = (float *) dst->data;- CUBLAS_CHECK(cublasSetStream(g_cublas_handles[g_main_device], main_stream));+ ggml_cuda_set_device(ctx.device);- //ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- //void * src0_ddq = src0_extra->data_device[g_main_device];- //half * src0_as_f16 = (half *) src0_ddq;+ // do the computation+ op(ctx, src0, src1, dst, src0_ddf, src1_ddf, dst_ddf, ctx.stream());+ CUDA_CHECK(cudaGetLastError());+}- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;- float * src1_ddf = (float *) src1_extra->data_device[g_main_device];+static void ggml_cuda_set_peer_access(const int n_tokens, int main_device) {+ static bool peer_access_enabled = false;- ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;- float * dst_ddf = (float *) dst_extra->data_device[g_main_device];+ const bool enable_peer_access = n_tokens <= GGML_CUDA_PEER_MAX_BATCH_SIZE;- // convert src1 to fp16- const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);- GGML_ASSERT(to_fp16_cuda != nullptr);+ if (peer_access_enabled == enable_peer_access) {+ return;+ }- size_t src1_as = 0;- half * src1_as_f16 = (half *) ggml_cuda_pool_malloc(ne1 * sizeof(half), &src1_as);- to_fp16_cuda(src1_ddf, src1_as_f16, ne1, main_stream);+#ifdef NDEBUG+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ ggml_cuda_set_device(id);+ CUDA_CHECK(cudaDeviceSynchronize());+ }- size_t dst_as = 0;- half * dst_f16 = (half *) ggml_cuda_pool_malloc(ne * sizeof(half), &dst_as);+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ ggml_cuda_set_device(id);- GGML_ASSERT(ne12 % ne02 == 0);- GGML_ASSERT(ne13 % ne03 == 0);+ for (int id_other = 0; id_other < ggml_backend_cuda_get_device_count(); ++id_other) {+ if (id == id_other) {+ continue;+ }+ if (id != main_device && id_other != main_device) {+ continue;+ }- // broadcast factors- const int64_t r2 = ne12/ne02;- const int64_t r3 = ne13/ne03;+ int can_access_peer;+ CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, id_other));+ if (can_access_peer) {+ if (enable_peer_access) {+ cudaError_t err = cudaDeviceEnablePeerAccess(id_other, 0);+ if (err != cudaErrorPeerAccessAlreadyEnabled) {+ CUDA_CHECK(err);+ }+ } else {+ cudaError_t err = cudaDeviceDisablePeerAccess(id_other);+ if (err != cudaErrorPeerAccessNotEnabled) {+ CUDA_CHECK(err);+ }+ }+ }+ }+ }+#endif // NDEBUG- const half alpha_f16 = 1.0f;- const half beta_f16 = 0.0f;+ peer_access_enabled = enable_peer_access;- // use cublasGemmBatchedEx- const int ne23 = ne12*ne13;+ GGML_UNUSED(main_device);+}- const void ** ptrs_src = nullptr;- void ** ptrs_dst = nullptr;+static void ggml_cuda_op_mul_mat(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op,+ const bool convert_src1_to_q8_1) {- size_t ptrs_src_s = 0;- size_t ptrs_dst_s = 0;+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int64_t ne02 = src0->ne[2];+ const int64_t ne03 = src0->ne[3];++ const int64_t ne10 = src1->ne[0];+ const int64_t ne11 = src1->ne[1];+ const int64_t ne12 = src1->ne[2];+ const int64_t ne13 = src1->ne[3];+ const int64_t nrows1 = ggml_nrows(src1);++ GGML_ASSERT(ne03 == ne13);- ptrs_src = (const void **) ggml_cuda_pool_malloc(2*ne23*sizeof(void *), &ptrs_src_s);- ptrs_dst = ( void **) ggml_cuda_pool_malloc(1*ne23*sizeof(void *), &ptrs_dst_s);+ const int64_t ne0 = dst->ne[0];+ const int64_t ne1 = dst->ne[1];- int64_t src0_ne = ggml_nelements(src00);- half * src0_as_f16 = nullptr;- size_t src0_as = 0;- if (src00->type != GGML_TYPE_F16) {- src0_as_f16 = (half *) ggml_cuda_pool_malloc(src0_ne * sizeof(half), &src0_as);- }+ const int nb2 = dst->nb[2];+ const int nb3 = dst->nb[3];- static_assert(GGML_MAX_SRC == 6, "GGML_MAX_SRC == 6");- dim3 block_dims(ne13, ne12);- k_compute_batched_ptrs_id<<<1, block_dims, 0, main_stream>>>(- ptrs_src, ptrs_dst,- ne12, ne13,- ne23,- ne00*ne01*sizeof(half), ne00*ne01*ne02*sizeof(half),- nb12, nb13,- dst->nb[2], dst->nb[3],- r2, r3,- src00->type, src0_as_f16, src0_ne,- src1_as_f16, dst_f16,- (const int *)((ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device], id,- dst->src[2] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[2]->extra)->data_device[g_main_device] : nullptr,- dst->src[3] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[3]->extra)->data_device[g_main_device] : nullptr,- dst->src[4] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[4]->extra)->data_device[g_main_device] : nullptr,- dst->src[5] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[5]->extra)->data_device[g_main_device] : nullptr- );- CUDA_CHECK(cudaGetLastError());+ GGML_ASSERT(ggml_backend_buffer_is_cuda(dst->buffer));+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src1->buffer));+ ggml_backend_cuda_buffer_context * src1_ctx = (ggml_backend_cuda_buffer_context *) src1->buffer->context;+ ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *) dst->buffer->context;- CUBLAS_CHECK(- cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,- ne01, ne11, ne10,- &alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, ne00,- (const void **) (ptrs_src + 1*ne23), CUDA_R_16F, ne10,- &beta_f16, ( void **) (ptrs_dst + 0*ne23), CUDA_R_16F, ne01,- ne23,- CUBLAS_COMPUTE_16F,- CUBLAS_GEMM_DEFAULT_TENSOR_OP));+ GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1));- if (src0_as != 0) {- ggml_cuda_pool_free(src0_as_f16, src0_as);- }- if (ptrs_src_s != 0) {- ggml_cuda_pool_free(ptrs_src, ptrs_src_s);- }- if (ptrs_dst_s != 0) {- ggml_cuda_pool_free(ptrs_dst, ptrs_dst_s);- }+ GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0);- const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);- to_fp32_cuda(dst_f16, dst_ddf, ne, main_stream);+ const int64_t i02_divisor = ne12 / ne02;- ggml_cuda_pool_free(src1_as_f16, src1_as);- ggml_cuda_pool_free(dst_f16, dst_as);-}-#endif+ const size_t src0_ts = ggml_type_size(src0->type);+ const size_t src0_bs = ggml_blck_size(src0->type);+ const size_t q8_1_ts = sizeof(block_q8_1);+ const size_t q8_1_bs = QK8_1;-static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {-#if 0- ggml_cuda_mul_mat_id_cublas(dst);- // TODO: mmq/mmv support-#endif+ const bool src0_is_contiguous = ggml_is_contiguous(src0);+ const bool src1_is_contiguous = ggml_is_contiguous(src1);- const size_t nb11 = src1->nb[1];- const size_t nb1 = dst->nb[1];+ const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING);- const struct ggml_tensor * ids = src0;- const int32_t id = ((int32_t *) dst->op_params)[0];- const int32_t n_as = ((int32_t *) dst->op_params)[1];+ const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer);+ GGML_ASSERT(!(split && ne02 > 1));+ GGML_ASSERT(!(split && ne03 > 1));+ GGML_ASSERT(!(split && ne02 < ne12));- std::vector<char> ids_host(ggml_nbytes(ids));+ ggml_tensor_extra_gpu * src0_extra = split ? (ggml_tensor_extra_gpu *) src0->extra : nullptr;- cudaStream_t stream = g_cudaStreams[g_main_device][0];- if (ids->backend == GGML_BACKEND_TYPE_GPU) {- const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device];- CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream));- CUDA_CHECK(cudaStreamSynchronize(stream));- } else {- memcpy(ids_host.data(), ids->data, ggml_nbytes(ids));+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;+ if (split) {+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;+ tensor_split = buft_ctx->tensor_split;}- const ggml_tensor_extra_gpu * src1_extra = (const ggml_tensor_extra_gpu *) src1->extra;- const ggml_tensor_extra_gpu * dst_extra = (const ggml_tensor_extra_gpu *) dst->extra;+ struct dev_data {+ ggml_cuda_pool_alloc<char> src0_dd_alloc;+ ggml_cuda_pool_alloc<float> src1_ddf_alloc;+ ggml_cuda_pool_alloc<char> src1_ddq_alloc;+ ggml_cuda_pool_alloc<float> dst_dd_alloc;- ggml_tensor_extra_gpu src1_row_extra;- ggml_tensor_extra_gpu dst_row_extra;+ char * src0_dd = nullptr;+ float * src1_ddf = nullptr; // float+ char * src1_ddq = nullptr; // q8_1+ float * dst_dd = nullptr;- ggml_tensor src1_row = *src1;- ggml_tensor dst_row = *dst;+ int64_t row_low;+ int64_t row_high;+ };- src1_row.backend = GGML_BACKEND_TYPE_GPU;- dst_row.backend = GGML_BACKEND_TYPE_GPU;+ dev_data dev[GGML_CUDA_MAX_DEVICES];- src1_row.extra = &src1_row_extra;- dst_row.extra = &dst_row_extra;+ int used_devices = 0;- char * src1_original = src1->backend == GGML_BACKEND_TYPE_CPU ?- (char *) src1->data : (char *) src1_extra->data_device[g_main_device];- char * dst_original = dst->backend == GGML_BACKEND_TYPE_CPU ?- (char *) dst->data : (char *) dst_extra->data_device[g_main_device];+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ // by default, use all rows+ dev[id].row_low = 0;+ dev[id].row_high = ne01;- if (src1->ne[1] == 1) {- GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU);- GGML_ASSERT(dst->backend == GGML_BACKEND_TYPE_GPU);+ // for multi GPU, get the row boundaries from tensor split+ // and round to mul_mat_q tile sizes+ if (split) {+ const int64_t rounding = get_row_rounding(src0->type, tensor_split);- for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {- //int32_t row_id;- //CUDA_CHECK(cudaMemcpyAsync(&row_id, ids_dev + i01*ids->nb[1] + id*ids->nb[0], sizeof(int32_t), cudaMemcpyDeviceToHost, g_cudaStreams[g_main_device][0]));- //CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[g_main_device][0]));+ if (id != 0) {+ dev[id].row_low = ne01*tensor_split[id];+ if (dev[id].row_low < ne01) {+ dev[id].row_low -= dev[id].row_low % rounding;+ }+ }- const int32_t row_id = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);+ if (id != ggml_backend_cuda_get_device_count() - 1) {+ dev[id].row_high = ne01*tensor_split[id + 1];+ if (dev[id].row_high < ne01) {+ dev[id].row_high -= dev[id].row_high % rounding;+ }+ }+ }+ }- GGML_ASSERT(row_id >= 0 && row_id < n_as);+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {+ continue;+ }- const struct ggml_tensor * src0_row = dst->src[row_id + 2];+ used_devices++;- src1_row_extra.data_device[g_main_device] = src1_original + i01*src1->nb[1];- src1_row.data = (char *) src1->data + i01*src1->nb[1]; // TODO why is this set?+ const bool src1_on_device = id == src1_ctx->device;+ const bool dst_on_device = id == dst_ctx->device;- dst_row_extra.data_device[g_main_device] = dst_original + i01*dst->nb[1];- dst_row.data = (char *) dst->data + i01*dst->nb[1]; // TODO why is this set?+ ggml_cuda_set_device(id);+ cudaStream_t stream = ctx.stream(id, 0);- ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row);+ if (src0_is_contiguous) {+ dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data;+ } else {+ dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), ggml_nbytes(src0));}- } else {- cuda_pool_alloc<char> src1_contiguous(sizeof(float)*ggml_nelements(src1));- cuda_pool_alloc<char> dst_contiguous(sizeof(float)*ggml_nelements(dst));- src1_row_extra.data_device[g_main_device] = src1_contiguous.get();- dst_row_extra.data_device[g_main_device] = dst_contiguous.get();+ if (src1_on_device && src1_is_contiguous) {+ dev[id].src1_ddf = (float *) src1->data;+ } else {+ dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1));+ }- const cudaMemcpyKind src1_kind = src1->backend == GGML_BACKEND_TYPE_CPU ?- cudaMemcpyHostToDevice : cudaMemcpyDeviceToDevice;- const cudaMemcpyKind dst_kind = dst->backend == GGML_BACKEND_TYPE_CPU ?- cudaMemcpyDeviceToHost : cudaMemcpyDeviceToDevice;+ if (convert_src1_to_q8_1) {+ dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs);- for (int32_t row_id = 0; row_id < n_as; ++row_id) {- const struct ggml_tensor * src0_row = dst->src[row_id + 2];+ if (src1_on_device && src1_is_contiguous) {+ quantize_row_q8_1_cuda(dev[id].src1_ddf, dev[id].src1_ddq, ne10, nrows1, src1_padded_col_size, stream);+ CUDA_CHECK(cudaGetLastError());+ }+ }- int64_t num_src1_rows = 0;- for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {- const int32_t row_id_i = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);+ if (dst_on_device) {+ dev[id].dst_dd = (float *) dst->data;+ } else {+ const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst);+ dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(ctx.pool(id), size_dst_ddf);+ }+ }- if (row_id_i != row_id) {- continue;- }+ // if multiple devices are used they need to wait for the main device+ // here an event is recorded that signals that the main device has finished calculating the input data+ if (split && used_devices > 1) {+ ggml_cuda_set_device(ctx.device);+ CUDA_CHECK(cudaEventRecord(src0_extra->events[ctx.device][0], ctx.stream()));+ }- GGML_ASSERT(row_id >= 0 && row_id < n_as);+ const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11;+ for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) {+ const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_CUDA_MAX_STREAMS : 0;+ const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride;- CUDA_CHECK(cudaMemcpyAsync(src1_contiguous.get() + num_src1_rows*nb11, src1_original + i01*nb11,- nb11, src1_kind, stream));- num_src1_rows++;+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {+ continue;}- if (num_src1_rows == 0) {- continue;+ const bool src1_on_device = id == src1_ctx->device;+ const bool dst_on_device = id == dst_ctx->device;+ const int64_t row_diff = dev[id].row_high - dev[id].row_low;++ ggml_cuda_set_device(id);+ cudaStream_t stream = ctx.stream(id, is);++ // wait for main GPU data if necessary+ if (split && (id != ctx.device || is != 0)) {+ CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[ctx.device][0], 0));}- src1_row.ne[1] = num_src1_rows;- dst_row.ne[1] = num_src1_rows;+ for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) {+ const int64_t i03 = i0 / ne12;+ const int64_t i02 = i0 % ne12;- src1_row.nb[1] = nb11;- src1_row.nb[2] = num_src1_rows*nb11;- src1_row.nb[3] = num_src1_rows*nb11;+ const size_t src1_ddq_i_offset = (i0*ne11 + src1_col_0) * src1_padded_col_size*q8_1_ts/q8_1_bs;- dst_row.nb[1] = nb1;- dst_row.nb[2] = num_src1_rows*nb1;- dst_row.nb[3] = num_src1_rows*nb1;+ // for split tensors the data begins at i0 == i0_offset_low+ char * src0_dd_i = dev[id].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;+ float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10;+ char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset;+ float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);- ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row);+ // the main device memory buffer can be on VRAM scratch, with space for all partial results+ // in that case an offset on dst_ddf_i is needed+ if (id == ctx.device) {+ dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split+ }- num_src1_rows = 0;- for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {- const int32_t row_id_i = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);+ // copy src0, src1 to device if necessary+ if (src1_is_contiguous) {+ if (id != ctx.device) {+ if (convert_src1_to_q8_1) {+ char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset;+ CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddq_i, id, src1_ddq_i_source, ctx.device,+ src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream));+ } else {+ float * src1_ddf_i_source = (float *) src1->data;+ src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10;+ CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, ctx.device,+ src1_ncols*ne10*sizeof(float), stream));+ }+ }+ } else if (src1_on_device && !src1_is_contiguous) {+ CUDA_CHECK(ggml_cuda_cpy_tensor_2d(+ src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream));+ } else {+ GGML_ASSERT(false);+ }- if (row_id_i != row_id) {- continue;+ if (convert_src1_to_q8_1 && !src1_is_contiguous) {+ quantize_row_q8_1_cuda(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, src1_padded_col_size, stream);+ CUDA_CHECK(cudaGetLastError());}- GGML_ASSERT(row_id >= 0 && row_id < n_as);+ if (src1_col_0 == 0 && !src0_is_contiguous && i02 % i02_divisor == 0) {+ CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream));+ }- CUDA_CHECK(cudaMemcpyAsync(dst_original + i01*nb1, dst_contiguous.get() + num_src1_rows*nb1,- nb1, dst_kind, stream));- num_src1_rows++;+ // do the computation+ op(ctx, src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i,+ dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream);+ CUDA_CHECK(cudaGetLastError());++ // copy dst to host or other device if necessary+ if (!dst_on_device) {+ void * dst_off_device = dst->data;+ if (split) {+ // src0 = weight matrix is saved as a transposed matrix for better memory layout.+ // dst is NOT transposed.+ // The outputs of matrix matrix multiplications can therefore NOT simply be concatenated for >1 GPU.+ // Instead they need to be copied to the correct slice in ne0 = dst row index.+ // If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results.+ float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);+ GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));+ dhf_dst_i += src1_col_0*ne0 + dev[id].row_low;+#if !defined(GGML_USE_HIPBLAS)+ // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices+ cudaMemcpy3DPeerParms p = {};+ p.dstDevice = ctx.device;+ p.dstPtr = make_cudaPitchedPtr(dhf_dst_i, ne0*sizeof(float), row_diff, src1_ncols);+ p.srcDevice = id;+ p.srcPtr = make_cudaPitchedPtr(dst_dd_i, row_diff*sizeof(float), row_diff, src1_ncols);+ p.extent = make_cudaExtent(row_diff*sizeof(float), src1_ncols, 1);+ CUDA_CHECK(cudaMemcpy3DPeerAsync(&p, stream));+#else+ // HIP does not support cudaMemcpy3DPeerAsync or vmm pools+ CUDA_CHECK(cudaMemcpy2DAsync(dhf_dst_i, ne0*sizeof(float),+ dst_dd_i, row_diff*sizeof(float),+ row_diff*sizeof(float), src1_ncols,+ cudaMemcpyDeviceToDevice, stream));+#endif+ } else {+ float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);+ GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));+ dhf_dst_i += src1_col_0*ne0;+ CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), cudaMemcpyDeviceToDevice, stream));+ }+ }++ // add event for the main device to wait on until other device is done+ if (split && (id != ctx.device || is != 0)) {+ CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream));+ }}}}- if (dst->backend == GGML_BACKEND_TYPE_CPU) {- CUDA_CHECK(cudaStreamSynchronize(stream));+ // main device waits for all other devices to be finished+ if (split && ggml_backend_cuda_get_device_count() > 1) {+ int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE;+ is_max = is_max <= GGML_CUDA_MAX_STREAMS ? is_max : GGML_CUDA_MAX_STREAMS;++ ggml_cuda_set_device(ctx.device);+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ if (dev[id].row_low == dev[id].row_high) {+ continue;+ }+ for (int64_t is = 0; is < is_max; ++is) {+ CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(), src0_extra->events[id][is], 0));+ }+ }}}-static void ggml_cuda_scale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_scale);+static void ggml_cuda_repeat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_repeat);}-static void ggml_cuda_clamp(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_clamp);+static void ggml_cuda_get_rows(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_get_rows);}-static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- const int64_t ne = ggml_nelements(src0);- GGML_ASSERT(ne == ggml_nelements(src1));-- GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU);- GGML_ASSERT(src1->backend == GGML_BACKEND_TYPE_GPU);-- GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX);- GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX);-- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];+static void ggml_cuda_add(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_add);+}- //GGML_ASSERT(src0->ne[3] == 1);+static void ggml_cuda_acc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_acc);+}- const int64_t nb00 = src0->nb[0];- const int64_t nb01 = src0->nb[1];- const int64_t nb02 = src0->nb[2];- const int64_t nb03 = src0->nb[3];+static void ggml_cuda_mul(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_mul);+}- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];+static void ggml_cuda_div(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_div);+}- //GGML_ASSERT(src1->ne[3] == 1);+static void ggml_cuda_gelu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_gelu);+}- const int64_t nb10 = src1->nb[0];- const int64_t nb11 = src1->nb[1];- const int64_t nb12 = src1->nb[2];- const int64_t nb13 = src1->nb[3];+static void ggml_cuda_silu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_silu);+}- ggml_cuda_set_device(g_main_device);- cudaStream_t main_stream = g_cudaStreams[g_main_device][0];+static void ggml_cuda_gelu_quick(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_gelu_quick);+}- const ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- const ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;+static void ggml_cuda_tanh(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_tanh);+}- char * src0_ddc = (char *) src0_extra->data_device[g_main_device];- char * src1_ddc = (char *) src1_extra->data_device[g_main_device];+static void ggml_cuda_relu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_relu);+}- if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {- ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {- ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {- ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {- ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {- ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {- ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {- ggml_cpy_f16_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else {- fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,- ggml_type_name(src0->type), ggml_type_name(src1->type));- GGML_ASSERT(false);- }+static void ggml_cuda_hardsigmoid(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_hardsigmoid);+}- (void) dst;+static void ggml_cuda_hardswish(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_hardswish);+}+static void ggml_cuda_leaky_relu(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_leaky_relu);}-static void ggml_cuda_dup(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- // TODO: why do we pass dst as src1 here?- ggml_cuda_cpy(src0, dst, nullptr);- (void) src1;+static void ggml_cuda_sqr(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_sqr);}-static void ggml_cuda_diag_mask_inf(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_diag_mask_inf);+static void ggml_cuda_norm(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_norm);}-static void ggml_cuda_soft_max(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_soft_max);+static void ggml_cuda_group_norm(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_group_norm);}-static void ggml_cuda_rope(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- GGML_ASSERT(ggml_is_contiguous(src0)); // TODO: this restriction is temporary until non-cont support is implemented- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_rope);+static void ggml_cuda_concat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_concat);}-static void ggml_cuda_alibi(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_alibi);+static void ggml_cuda_upscale(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_upscale);}-static void ggml_cuda_pool2d(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_pool2d);+static void ggml_cuda_pad(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_pad);}-static void ggml_cuda_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_im2col);+static void ggml_cuda_arange(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_arange);}-static void ggml_cuda_sum_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- GGML_ASSERT(ggml_is_contiguous(src0));- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_sum_rows);+static void ggml_cuda_timestep_embedding(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_timestep_embedding);}-static void ggml_cuda_argsort(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- GGML_ASSERT(ggml_is_contiguous(src0));- ggml_cuda_op_flatten(src0, src1, dst, ggml_cuda_op_argsort);+static void ggml_cuda_rms_norm(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_rms_norm);}-static void ggml_cuda_nop(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- (void) src0;- (void) src1;- (void) dst;+static void ggml_cuda_mul_mat_vec_p021(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){+ GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1));+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));+ GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation+ GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // 0213 permutation+ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);++ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int64_t ne02 = src0->ne[2];++ const int64_t ne12 = src1->ne[2];++ ggml_cuda_set_device(ctx.device);+ cudaStream_t main_stream = ctx.stream();++ void * src0_ddq = src0->data;+ float * src1_ddf = (float *) src1->data;+ float * dst_ddf = (float *) dst->data;++ ggml_mul_mat_p021_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, ne02, ne12, main_stream);}-static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {- static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");+static void ggml_cuda_mul_mat_vec_nc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){+ GGML_ASSERT(!ggml_is_transposed(src0));+ GGML_ASSERT(!ggml_is_transposed(src1));+ GGML_ASSERT(!ggml_is_permuted(src0));+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));+ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);- return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]);+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int64_t ne02 = src0->ne[2];++ const int64_t nb01 = src0->nb[1];+ const int64_t nb02 = src0->nb[2];++ const int64_t ne12 = src1->ne[2];++ ggml_cuda_set_device(ctx.device);+ cudaStream_t main_stream = ctx.stream();++ void * src0_ddq = src0->data;+ float * src1_ddf = (float *) src1->data;+ float * dst_ddf = (float *) dst->data;++ const int64_t row_stride_x = nb01 / sizeof(half);+ const int64_t channel_stride_x = nb02 / sizeof(half);++ ggml_mul_mat_vec_nc_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, row_stride_x, ne02, ne12, channel_stride_x, main_stream);}-GGML_CALL static void ggml_cuda_set_main_device(const int main_device) {- if (main_device >= g_device_count) {- fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n",- main_device, g_device_count, g_main_device);+static __global__ void k_compute_batched_ptrs(+ const half * src0_as_f16, const half * src1_as_f16, char * dst,+ const void ** ptrs_src, void ** ptrs_dst,+ int64_t ne12, int64_t ne13,+ int64_t ne23,+ size_t nb02, size_t nb03,+ size_t nb12, size_t nb13,+ size_t nbd2, size_t nbd3,+ int64_t r2, int64_t r3) {+ int64_t i13 = blockIdx.x * blockDim.x + threadIdx.x;+ int64_t i12 = blockIdx.y * blockDim.y + threadIdx.y;++ if (i13 >= ne13 || i12 >= ne12) {return;}- if (g_main_device != main_device && g_device_count > 1) {- g_main_device = main_device;- //cudaDeviceProp prop;- //CUDA_CHECK(cudaGetDeviceProperties(&prop, g_main_device));- //fprintf(stderr, "%s: using device %d (%s) as main device\n", __func__, g_main_device, prop.name);- }+ int64_t i03 = i13 / r3;+ int64_t i02 = i12 / r2;++ ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03;+ ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12 + i13*nb13;+ ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;}-GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {- if (!g_cublas_loaded) return false;+static void ggml_cuda_mul_mat_batched_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_ASSERT(!ggml_is_transposed(src0));+ GGML_ASSERT(!ggml_is_transposed(src1));- ggml_cuda_func_t func;- const bool any_on_device = tensor->backend == GGML_BACKEND_TYPE_GPU- || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU || tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT))- || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_TYPE_GPU);+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));+ GGML_ASSERT(src0->type == GGML_TYPE_F16);- if (!any_on_device && tensor->op != GGML_OP_MUL_MAT && tensor->op != GGML_OP_MUL_MAT_ID) {- return false;- }+ GGML_TENSOR_BINARY_OP_LOCALS- if (tensor->op == GGML_OP_MUL_MAT) {- if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) {-#ifndef NDEBUG- fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]);-#endif- return false;- }- }+ const int64_t ne_dst = ggml_nelements(dst);- switch (tensor->op) {- case GGML_OP_REPEAT:- func = ggml_cuda_repeat;- break;- case GGML_OP_GET_ROWS:- func = ggml_cuda_get_rows;- break;- case GGML_OP_DUP:- func = ggml_cuda_dup;- break;- case GGML_OP_ADD:- func = ggml_cuda_add;- break;- case GGML_OP_ACC:- func = ggml_cuda_acc;- break;- case GGML_OP_MUL:- func = ggml_cuda_mul;- break;- case GGML_OP_DIV:- func = ggml_cuda_div;- break;- case GGML_OP_UNARY:- switch (ggml_get_unary_op(tensor)) {- case GGML_UNARY_OP_GELU:- func = ggml_cuda_gelu;- break;- case GGML_UNARY_OP_SILU:- func = ggml_cuda_silu;- break;- case GGML_UNARY_OP_GELU_QUICK:- func = ggml_cuda_gelu_quick;- break;- case GGML_UNARY_OP_TANH:- func = ggml_cuda_tanh;- break;- case GGML_UNARY_OP_RELU:- func = ggml_cuda_relu;- break;- case GGML_UNARY_OP_HARDSIGMOID:- func = ggml_cuda_hardsigmoid;- break;- case GGML_UNARY_OP_HARDSWISH:- func = ggml_cuda_hardswish;- break;- default:- return false;- }- break;- case GGML_OP_NORM:- func = ggml_cuda_norm;- break;- case GGML_OP_GROUP_NORM:- func = ggml_cuda_group_norm;- break;- case GGML_OP_CONCAT:- func = ggml_cuda_concat;- break;- case GGML_OP_UPSCALE:- func = ggml_cuda_upscale;- break;- case GGML_OP_PAD:- func = ggml_cuda_pad;- break;- case GGML_OP_LEAKY_RELU:- func = ggml_cuda_leaky_relu;- break;- case GGML_OP_RMS_NORM:- func = ggml_cuda_rms_norm;- break;- case GGML_OP_MUL_MAT:- if (!any_on_device && !ggml_cuda_can_mul_mat(tensor->src[0], tensor->src[1], tensor)) {- return false;- }- func = ggml_cuda_mul_mat;- break;- case GGML_OP_MUL_MAT_ID:- if (!any_on_device && !ggml_cuda_can_mul_mat(tensor->src[2], tensor->src[1], tensor)) {- return false;- }- func = ggml_cuda_mul_mat_id;- break;- case GGML_OP_SCALE:- func = ggml_cuda_scale;- break;- case GGML_OP_SQR:- func = ggml_cuda_sqr;- break;- case GGML_OP_CLAMP:- func = ggml_cuda_clamp;- break;- case GGML_OP_CPY:- func = ggml_cuda_cpy;- break;- case GGML_OP_CONT:- func = ggml_cuda_dup;- break;- case GGML_OP_NONE:- case GGML_OP_RESHAPE:- case GGML_OP_VIEW:- case GGML_OP_PERMUTE:- case GGML_OP_TRANSPOSE:- func = ggml_cuda_nop;- break;- case GGML_OP_DIAG_MASK_INF:- func = ggml_cuda_diag_mask_inf;- break;- case GGML_OP_SOFT_MAX:- func = ggml_cuda_soft_max;- break;- case GGML_OP_ROPE:- func = ggml_cuda_rope;- break;- case GGML_OP_ALIBI:- func = ggml_cuda_alibi;- break;- case GGML_OP_IM2COL:- func = ggml_cuda_im2col;- break;- case GGML_OP_POOL_2D:- func = ggml_cuda_pool2d;- break;- case GGML_OP_SUM_ROWS:- func = ggml_cuda_sum_rows;- break;- case GGML_OP_ARGSORT:- func = ggml_cuda_argsort;- break;- default:- return false;- }+ ggml_cuda_set_device(ctx.device);+ cudaStream_t main_stream = ctx.stream();++ CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream));- if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_TYPE_GPU_SPLIT) {- ggml_cuda_set_peer_access(tensor->src[1]->ne[1]);+ void * src0_ddq = src0->data;+ half * src0_f16 = (half *) src0_ddq;+ float * src1_ddf = (float *) src1->data;+ float * dst_ddf = (float *) dst->data;++ // convert src1 to fp16+ ggml_cuda_pool_alloc<half> src1_f16_alloc(ctx.pool());+ if (src1->type != GGML_TYPE_F16) {+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ const int64_t ne_src1 = ggml_nelements(src1);+ src1_f16_alloc.alloc(ne_src1);+ GGML_ASSERT(to_fp16_cuda != nullptr);+ to_fp16_cuda(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream);}+ half * src1_f16 = src1->type == GGML_TYPE_F16 ? (half *) src1_ddf : src1_f16_alloc.get();- if (params->ith != 0) {- return true;- }- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return true;- }- func(tensor->src[0], tensor->src[1], tensor);- return true;-}+ ggml_cuda_pool_alloc<half> dst_f16(ctx.pool());+ char * dst_t;-GGML_CALL int ggml_cuda_get_device_count() {- int device_count;- if (cudaGetDeviceCount(&device_count) != cudaSuccess) {- return 0;- }- return device_count;-}+ cublasComputeType_t cu_compute_type = CUBLAS_COMPUTE_16F;+ cudaDataType_t cu_data_type = CUDA_R_16F;-GGML_CALL void ggml_cuda_get_device_description(int device, char * description, size_t description_size) {- cudaDeviceProp prop;- CUDA_CHECK(cudaGetDeviceProperties(&prop, device));- snprintf(description, description_size, "%s", prop.name);-}+ // dst strides+ size_t nbd2 = dst->nb[2];+ size_t nbd3 = dst->nb[3];-////////////////////////////////////////////////////////////////////////////////+ const half alpha_f16 = 1.0f;+ const half beta_f16 = 0.0f;-// backend interface+ const float alpha_f32 = 1.0f;+ const float beta_f32 = 0.0f;-#define UNUSED GGML_UNUSED+ const void * alpha = &alpha_f16;+ const void * beta = &beta_f16;-struct ggml_backend_cuda_context {- int device;- std::string name;-};+ if (dst->op_params[0] == GGML_PREC_DEFAULT) {+ dst_t = (char *) dst_f16.alloc(ne_dst);-// cuda buffer+ nbd2 /= sizeof(float) / sizeof(half);+ nbd3 /= sizeof(float) / sizeof(half);+ } else {+ dst_t = (char *) dst_ddf;-struct ggml_backend_cuda_buffer_context {- int device;- void * dev_ptr = nullptr;- ggml_tensor_extra_gpu * temp_tensor_extras = nullptr;- size_t temp_tensor_extra_index = 0;- std::string name;+ cu_compute_type = CUBLAS_COMPUTE_32F;+ cu_data_type = CUDA_R_32F;- ggml_backend_cuda_buffer_context(int device, void * dev_ptr) :- device(device), dev_ptr(dev_ptr),- name(GGML_CUDA_NAME + std::to_string(device)) {+ alpha = &alpha_f32;+ beta = &beta_f32;}- ~ggml_backend_cuda_buffer_context() {- delete[] temp_tensor_extras;- }+ GGML_ASSERT(ne12 % ne02 == 0);+ GGML_ASSERT(ne13 % ne03 == 0);- ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() {- // TODO: remove GGML_CUDA_MAX_NODES, allocate dynamically and reuse in backend_buffer_reset- if (temp_tensor_extras == nullptr) {- temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES];- }+ // broadcast factors+ const int64_t r2 = ne12/ne02;+ const int64_t r3 = ne13/ne03;- size_t alloc_index = temp_tensor_extra_index;- temp_tensor_extra_index = (temp_tensor_extra_index + 1) % GGML_CUDA_MAX_NODES;- ggml_tensor_extra_gpu * extra = &temp_tensor_extras[alloc_index];- memset(extra, 0, sizeof(*extra));+#if 0+ // use cublasGemmEx+ {+ for (int i13 = 0; i13 < ne13; ++i13) {+ for (int i12 = 0; i12 < ne12; ++i12) {+ int i03 = i13 / r3;+ int i02 = i12 / r2;- return extra;+ CUBLAS_CHECK(+ cublasGemmEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,+ ne01, ne11, ne10,+ alpha, (const char *) src0_as_f16 + i02*src0->nb[2] + i03*src0->nb[3] , CUDA_R_16F, nb01/sizeof(half),+ (const char *) src1_as_f16 + i12*src1->nb[2]/2 + i13*src1->nb[3]/2, CUDA_R_16F, nb11/sizeof(float),+ beta, ( char *) dst_t + i12*nbd2 + i13*nbd3, cu_data_type, ne01,+ cu_compute_type,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));+ }+ }}-};+#else+ if (r2 == 1 && r3 == 1 && src0->nb[2]*src0->ne[2] == src0->nb[3] && src1->nb[2]*src1->ne[2] == src1->nb[3]) {+ // there is no broadcast and src0, src1 are contiguous across dims 2, 3+ // use cublasGemmStridedBatchedEx+ CUBLAS_CHECK(+ cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,+ ne01, ne11, ne10,+ alpha, (const char *) src0_f16, CUDA_R_16F, nb01/nb00, nb02/nb00, // strideA+ (const char *) src1_f16, CUDA_R_16F, nb11/nb10, nb12/nb10, // strideB+ beta, ( char *) dst_t, cu_data_type, ne01, nb2/nb0, // strideC+ ne12*ne13,+ cu_compute_type,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));+ } else {+ // use cublasGemmBatchedEx+ const int ne23 = ne12*ne13;-GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;- return ctx->name.c_str();-}+ ggml_cuda_pool_alloc<const void *> ptrs_src(ctx.pool(), 2*ne23);+ ggml_cuda_pool_alloc< void *> ptrs_dst(ctx.pool(), 1*ne23);-GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {- return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name;-}+ dim3 block_dims(ne13, ne12);+ k_compute_batched_ptrs<<<1, block_dims, 0, main_stream>>>(+ src0_f16, src1_f16, dst_t,+ ptrs_src.get(), ptrs_dst.get(),+ ne12, ne13,+ ne23,+ nb02, nb03,+ src1->type == GGML_TYPE_F16 ? nb12 : nb12/2,+ src1->type == GGML_TYPE_F16 ? nb13 : nb13/2,+ nbd2, nbd3,+ r2, r3);+ CUDA_CHECK(cudaGetLastError());-GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;- CUDA_CHECK(cudaFree(ctx->dev_ptr));- delete ctx;-}+ CUBLAS_CHECK(+ cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,+ ne01, ne11, ne10,+ alpha, (const void **) (ptrs_src.get() + 0*ne23), CUDA_R_16F, nb01/nb00,+ (const void **) (ptrs_src.get() + 1*ne23), CUDA_R_16F, nb11/nb10,+ beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne01,+ ne23,+ cu_compute_type,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));+ }+#endif-GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;- return ctx->dev_ptr;+ if (dst->op_params[0] == GGML_PREC_DEFAULT) {+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);+ to_fp32_cuda(dst_f16.get(), dst_ddf, ne_dst, main_stream);+ }}-GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer);- if (tensor->view_src != NULL && tensor->view_offs == 0) {- assert(tensor->view_src->buffer->buft == buffer->buft);- tensor->backend = tensor->view_src->backend;- tensor->extra = tensor->view_src->extra;- return;+ int64_t min_compute_capability = INT_MAX;++ bool any_pascal_with_slow_fp16 = false;+ if (split) {+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;+ auto & tensor_split = buft_ctx->tensor_split;+ for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {+ // skip devices that are not going to do any work:+ if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {+ continue;+ }++ if (min_compute_capability > get_cuda_global_info().devices[id].cc) {+ min_compute_capability = get_cuda_global_info().devices[id].cc;+ }+ if (get_cuda_global_info().devices[id].cc == 610) {+ any_pascal_with_slow_fp16 = true;+ }+ }+ } else {+ min_compute_capability = get_cuda_global_info().devices[ctx.device].cc;+ any_pascal_with_slow_fp16 = get_cuda_global_info().devices[ctx.device].cc == 610;}- ggml_tensor_extra_gpu * extra = ctx->ggml_cuda_alloc_temp_tensor_extra();+ // check data types and tensor shapes for custom matrix multiplication kernels:+ bool use_dequantize_mul_mat_vec = (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16)+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32+ && src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->ne[1] == 1;- extra->data_device[ctx->device] = tensor->data;+ bool use_mul_mat_vec_q = ggml_is_quantized(src0->type)+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32+ && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;- tensor->backend = GGML_BACKEND_TYPE_GPU;- tensor->extra = extra;+ bool use_mul_mat_q = ggml_cuda_supports_mmq(src0->type)+ && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;- if (ggml_is_quantized(tensor->type)) {- // initialize padding to 0 to avoid possible NaN values- size_t original_size = ggml_nbytes(tensor);- size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- if (padded_size > original_size && tensor->view_src == nullptr) {- CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));- }- }-}+ const bool fp16_performance_good = min_compute_capability >= CC_RDNA1;-GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU);+#ifdef CUDA_USE_TENSOR_CORES+ use_mul_mat_q = use_mul_mat_q && min_compute_capability < CC_RDNA3;+#endif // CUDA_USE_TENSOR_CORES- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+#else- ggml_cuda_set_device(ctx->device);- CUDA_CHECK(cudaDeviceSynchronize());- CUDA_CHECK(cudaMemcpy((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice));- CUDA_CHECK(cudaDeviceSynchronize());-}+ // fp16 performance is good on Volta or newer and on P100 (compute capability 6.0)+ const bool fp16_performance_good = min_compute_capability >= CC_PASCAL && !any_pascal_with_slow_fp16;-GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {- GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU);+ // mmvq and mmq need the __dp4a instruction which on NVIDIA is only available for CC >= 6.1+ use_mul_mat_vec_q = use_mul_mat_vec_q && min_compute_capability >= MIN_CC_DP4A;+ use_mul_mat_q = use_mul_mat_q && min_compute_capability >= MIN_CC_DP4A;- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+#ifdef CUDA_USE_TENSOR_CORES+ // when tensor cores are available, use them for large batch size+ // ref: https://github.com/ggerganov/llama.cpp/pull/3776+ use_mul_mat_q = use_mul_mat_q && (!fp16_performance_good || src1->ne[1] <= MMQ_MAX_BATCH_SIZE);+#endif // CUDA_USE_TENSOR_CORES- ggml_cuda_set_device(ctx->device);- CUDA_CHECK(cudaDeviceSynchronize());- CUDA_CHECK(cudaMemcpy(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost));- CUDA_CHECK(cudaDeviceSynchronize());+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)++ // if mmvq is available it's a better choice than dmmv:+#ifndef GGML_CUDA_FORCE_DMMV+ use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q;+#endif // GGML_CUDA_FORCE_DMMV++ // debug helpers+ //printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);+ //printf(" %8d %8d %8d %8d\n", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]);+ //printf("src1: %8d %8d %8d %8d\n", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]);+ //printf(" %8d %8d %8d %8d\n", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]);+ //printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);+ //printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);++ if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {+ // KQ single-batch+ ggml_cuda_mul_mat_vec_p021(ctx, src0, src1, dst);+ } else if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {+ // KQV single-batch+ ggml_cuda_mul_mat_vec_nc(ctx, src0, src1, dst);+ } else if (!split && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {+ // KQ + KQV multi-batch+ ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst);+ } else if (use_dequantize_mul_mat_vec) {+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false);+ } else if (use_mul_mat_vec_q) {+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, true);+ } else if (use_mul_mat_q) {+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, true);+ } else {+ ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false);+ }}-GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {- if (ggml_backend_buffer_is_cuda(src->buffer)) {- ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;- ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+#if 0+template<typename ... Srcs>+static __global__ void k_compute_batched_ptrs_id(+ const void ** ptrs_src, void ** ptrs_dst,+ int ne12, int ne13,+ int ne23,+ int nb02, int nb03,+ int nb12, int nb13,+ int nb2, int nb3,+ int r2, int r3,+ ggml_type src0_type, half * src0_as_f16, int64_t src0_ne,+ const half * src1_f16, half * dst_f16,+ const int32_t * ids, const int id,+ Srcs... src0s) {++ int i = ids[id];++ half * src0_f16;+ const void * srcs_ar[] = { (const half *) src0s... };+ if (src0_type == GGML_TYPE_F16) {+ src0_f16 = (half *) srcs_ar[i];+ } else {+ src0_f16 = src0_as_f16;+ if (threadIdx.x == 0 && threadIdx.y == 0) {+ const to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(src0_type);+ to_fp16(srcs_ar[i], src0_f16, src0_ne, cudaStreamFireAndForget);+ }+ }- ggml_cuda_set_device(src_ctx->device);- CUDA_CHECK(cudaDeviceSynchronize());- ggml_cuda_set_device(dst_ctx->device);- CUDA_CHECK(cudaDeviceSynchronize());- CUDA_CHECK(cudaMemcpy((char *)dst->data, (const char *)src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice));- CUDA_CHECK(cudaDeviceSynchronize());+ int i13 = blockIdx.x * blockDim.x + threadIdx.x;+ int i12 = blockIdx.y * blockDim.y + threadIdx.y;- return true;+ if (i13 >= ne13 || i12 >= ne12) {+ return;}- return false;-}-GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ int i03 = i13 / r3;+ int i02 = i12 / r2;- ggml_cuda_set_device(ctx->device);- CUDA_CHECK(cudaDeviceSynchronize());- CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size));- CUDA_CHECK(cudaDeviceSynchronize());+ ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_f16 + i02*nb02 + i03*nb03;+ ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_f16 + i12*nb12/2 + i13*nb13/2;+ ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst_f16 + i12* nb2/2 + i13* nb3/2;}-static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {- /* .get_name = */ ggml_backend_cuda_buffer_get_name,- /* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,- /* .get_base = */ ggml_backend_cuda_buffer_get_base,- /* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor,- /* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor,- /* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,- /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,- /* .clear = */ ggml_backend_cuda_buffer_clear,- /* .reset = */ NULL,-};--// cuda buffer type-struct ggml_backend_cuda_buffer_type_context {- int device;- std::string name;-};--GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) {- ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;+static void ggml_cuda_mul_mat_id_cublas(ggml_tensor * dst) {+ const struct ggml_tensor * ids = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];+ const struct ggml_tensor * src00 = dst->src[2];- return ctx->name.c_str();-}+ const int id = dst->op_params[0];-GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {- ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;+ GGML_ASSERT(!ggml_is_transposed(src00));+ GGML_ASSERT(!ggml_is_transposed(src1));- ggml_cuda_set_device(buft_ctx->device);+ GGML_ASSERT(src00->backend != GGML_BACKEND_TYPE_GPU_SPLIT);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);- size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0+ const int64_t ne00 = src00->ne[0]; GGML_UNUSED(ne00);+ const int64_t ne01 = src00->ne[1];+ const int64_t ne02 = src00->ne[2];+ const int64_t ne03 = src00->ne[3];- void * dev_ptr;- cudaError_t err = cudaMalloc(&dev_ptr, size);- if (err != cudaSuccess) {- fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err));- return nullptr;- }+ //const int64_t nb01 = src00->nb[1];+ const int64_t nb02 = src00->nb[2]; GGML_UNUSED(nb02);+ const int64_t nb03 = src00->nb[3]; GGML_UNUSED(nb03);- ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr);+ const int64_t ne10 = src1->ne[0];+ const int64_t ne11 = src1->ne[1];+ const int64_t ne12 = src1->ne[2];+ const int64_t ne13 = src1->ne[3];- return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);-}+ //const int64_t nb11 = src1->nb[1];+ const int64_t nb12 = src1->nb[2]; GGML_UNUSED(nb12);+ const int64_t nb13 = src1->nb[3]; GGML_UNUSED(nb13);-GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {- return 128;+ const int64_t ne1 = ggml_nelements(src1);+ const int64_t ne = ggml_nelements(dst);- UNUSED(buft);-}+ ggml_cuda_set_device(g_main_device);+ cudaStream_t main_stream = g_cudaStreams[g_main_device][0];-GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {- size_t size = ggml_nbytes(tensor);- int64_t ne0 = tensor->ne[0];+ CUBLAS_CHECK(cublasSetStream(g_cublas_handles[g_main_device], main_stream));- if (ggml_is_quantized(tensor->type)) {- if (ne0 % MATRIX_ROW_PADDING != 0) {- size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);- }- }+ //ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;+ //void * src0_ddq = src0_extra->data_device[g_main_device];+ //half * src0_as_f16 = (half *) src0_ddq;- return size;+ ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;+ float * src1_ddf = (float *) src1_extra->data_device[g_main_device];- UNUSED(buft);-}+ ggml_tensor_extra_gpu * dst_extra = (ggml_tensor_extra_gpu *) dst->extra;+ float * dst_ddf = (float *) dst_extra->data_device[g_main_device];-GGML_CALL static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {- if (!ggml_backend_is_cuda(backend)) {- return false;- }+ // convert src1 to fp16+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ GGML_ASSERT(to_fp16_cuda != nullptr);- ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;- ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;+ size_t src1_as = 0;+ half * src1_as_f16 = (half *) ggml_cuda_pool_malloc(ne1 * sizeof(half), &src1_as);+ to_fp16_cuda(src1_ddf, src1_as_f16, ne1, main_stream);- return buft_ctx->device == cuda_ctx->device;-}+ size_t dst_as = 0;+ half * dst_f16 = (half *) ggml_cuda_pool_malloc(ne * sizeof(half), &dst_as);-static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {- /* .get_name = */ ggml_backend_cuda_buffer_type_name,- /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,- /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,- /* .get_max_size = */ NULL, // defaults to SIZE_MAX- /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,- /* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend,- /* .is_host = */ NULL,-};+ GGML_ASSERT(ne12 % ne02 == 0);+ GGML_ASSERT(ne13 % ne03 == 0);-GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {- // FIXME: this is not thread safe- if (device >= ggml_backend_cuda_get_device_count()) {- return nullptr;- }+ // broadcast factors+ const int64_t r2 = ne12/ne02;+ const int64_t r3 = ne13/ne03;- static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];+ const half alpha_f16 = 1.0f;+ const half beta_f16 = 0.0f;- static bool ggml_backend_cuda_buffer_type_initialized = false;+ // use cublasGemmBatchedEx+ const int ne23 = ne12*ne13;- if (!ggml_backend_cuda_buffer_type_initialized) {- for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) {- ggml_backend_cuda_buffer_types[i] = {- /* .iface = */ ggml_backend_cuda_buffer_type_interface,- /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)},- };- }- ggml_backend_cuda_buffer_type_initialized = true;- }+ const void ** ptrs_src = nullptr;+ void ** ptrs_dst = nullptr;- return &ggml_backend_cuda_buffer_types[device];-}+ size_t ptrs_src_s = 0;+ size_t ptrs_dst_s = 0;-// cuda split buffer+ ptrs_src = (const void **) ggml_cuda_pool_malloc(2*ne23*sizeof(void *), &ptrs_src_s);+ ptrs_dst = ( void **) ggml_cuda_pool_malloc(1*ne23*sizeof(void *), &ptrs_dst_s);-struct ggml_backend_cuda_split_buffer_context {- ~ggml_backend_cuda_split_buffer_context() {- for (ggml_tensor_extra_gpu * extra : tensor_extras) {- for (int id = 0; id < g_device_count; ++id) {- for (int64_t is = 0; is < MAX_STREAMS; ++is) {- if (extra->events[id][is] != nullptr) {- CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));- }- }- if (extra->data_device[id] != nullptr) {- CUDA_CHECK(cudaFree(extra->data_device[id]));- }- }- delete extra;- }+ int64_t src0_ne = ggml_nelements(src00);+ half * src0_as_f16 = nullptr;+ size_t src0_as = 0;+ if (src00->type != GGML_TYPE_F16) {+ src0_as_f16 = (half *) ggml_cuda_pool_malloc(src0_ne * sizeof(half), &src0_as);}- std::vector<ggml_tensor_extra_gpu *> tensor_extras;-};--GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) {- return GGML_CUDA_NAME "_Split";-- UNUSED(buffer);-}+ static_assert(GGML_MAX_SRC == 6, "GGML_MAX_SRC == 6");+ dim3 block_dims(ne13, ne12);+ k_compute_batched_ptrs_id<<<1, block_dims, 0, main_stream>>>(+ ptrs_src, ptrs_dst,+ ne12, ne13,+ ne23,+ ne00*ne01*sizeof(half), ne00*ne01*ne02*sizeof(half),+ nb12, nb13,+ dst->nb[2], dst->nb[3],+ r2, r3,+ src00->type, src0_as_f16, src0_ne,+ src1_as_f16, dst_f16,+ (const int *)((ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device], id,+ dst->src[2] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[2]->extra)->data_device[g_main_device] : nullptr,+ dst->src[3] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[3]->extra)->data_device[g_main_device] : nullptr,+ dst->src[4] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[4]->extra)->data_device[g_main_device] : nullptr,+ dst->src[5] ? (const half *)((ggml_tensor_extra_gpu *)dst->src[5]->extra)->data_device[g_main_device] : nullptr+ );+ CUDA_CHECK(cudaGetLastError());-static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {- return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name;- UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds-}+ CUBLAS_CHECK(+ cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,+ ne01, ne11, ne10,+ &alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, ne00,+ (const void **) (ptrs_src + 1*ne23), CUDA_R_16F, ne10,+ &beta_f16, ( void **) (ptrs_dst + 0*ne23), CUDA_R_16F, ne01,+ ne23,+ CUBLAS_COMPUTE_16F,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP));-GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {- ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;- delete ctx;-}+ if (src0_as != 0) {+ ggml_cuda_pool_free(src0_as_f16, src0_as);+ }+ if (ptrs_src_s != 0) {+ ggml_cuda_pool_free(ptrs_src, ptrs_src_s);+ }+ if (ptrs_dst_s != 0) {+ ggml_cuda_pool_free(ptrs_dst, ptrs_dst_s);+ }-GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {- // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced- return (void *)0x1000;+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);+ to_fp32_cuda(dst_f16, dst_ddf, ne, main_stream);- UNUSED(buffer);+ ggml_cuda_pool_free(src1_as_f16, src1_as);+ ggml_cuda_pool_free(dst_f16, dst_as);}+#endif-GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {- GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported-- ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;- ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;-- const int64_t ne0 = tensor->ne[0];+static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+#if 0+ ggml_cuda_mul_mat_id_cublas(dst);+ // TODO: mmq/mmv support+#endif+ cudaStream_t stream = ctx.stream();- ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};+ const size_t nb11 = src1->nb[1];+ const size_t nb1 = dst->nb[1];- ctx->tensor_extras.push_back(extra);+ const struct ggml_tensor * ids = src0;+ const int32_t id = ((int32_t *) dst->op_params)[0];+ const int32_t n_as = ((int32_t *) dst->op_params)[1];- for (int id = 0; id < g_device_count; ++id) {- int64_t row_low, row_high;- get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);+ std::vector<char> ids_host(ggml_nbytes(ids));+ const char * ids_dev = (const char *) ids->data;+ CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream));+ CUDA_CHECK(cudaStreamSynchronize(stream));- int64_t nrows_split = row_high - row_low;- if (nrows_split == 0) {- continue;- }+ ggml_tensor src1_row = *src1;+ ggml_tensor dst_row = *dst;- size_t size = ggml_nbytes_split(tensor, nrows_split);- const size_t original_size = size;+ char * src1_original = (char *) src1->data;+ char * dst_original = (char *) dst->data;- // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses- if (ne0 % MATRIX_ROW_PADDING != 0) {- size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);- }+ if (src1->ne[1] == 1) {+ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ const int32_t row_id = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);- // FIXME: do not crash if cudaMalloc fails- // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first- ggml_cuda_set_device(id);- char * buf;- CUDA_CHECK(cudaMalloc(&buf, size));+ GGML_ASSERT(row_id >= 0 && row_id < n_as);- // set padding to 0 to avoid possible NaN values- if (size > original_size) {- CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size));- }+ const struct ggml_tensor * src0_row = dst->src[row_id + 2];- extra->data_device[id] = buf;+ src1_row.data = src1_original + i01*src1->nb[1];+ dst_row.data = dst_original + i01*dst->nb[1];- for (int64_t is = 0; is < MAX_STREAMS; ++is) {- CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming));+ ggml_cuda_mul_mat(ctx, src0_row, &src1_row, &dst_row);}- }- tensor->backend = GGML_BACKEND_TYPE_GPU_SPLIT;- tensor->extra = extra;-}+ } else {+ ggml_cuda_pool_alloc<char> src1_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(src1));+ ggml_cuda_pool_alloc<char> dst_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(dst));-GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- // split tensors must always be set in their entirety at once- GGML_ASSERT(offset == 0);- GGML_ASSERT(size == ggml_nbytes(tensor));+ src1_row.data = src1_contiguous.get();+ dst_row.data = dst_contiguous.get();- ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;+ for (int32_t row_id = 0; row_id < n_as; ++row_id) {+ const struct ggml_tensor * src0_row = dst->src[row_id + 2];- const int64_t ne0 = tensor->ne[0];- const size_t nb1 = tensor->nb[1];- ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;+ int64_t num_src1_rows = 0;+ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ const int32_t row_id_i = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);- for (int id = 0; id < g_device_count; ++id) {- int64_t row_low, row_high;- get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);+ if (row_id_i != row_id) {+ continue;+ }- int64_t nrows_split = row_high - row_low;- if (nrows_split == 0) {- continue;- }+ GGML_ASSERT(row_id >= 0 && row_id < n_as);- const size_t offset_split = row_low*nb1;- size_t size = ggml_nbytes_split(tensor, nrows_split);- const size_t original_size = size;+ CUDA_CHECK(cudaMemcpyAsync(src1_contiguous.get() + num_src1_rows*nb11, src1_original + i01*nb11,+ nb11, cudaMemcpyDeviceToDevice, stream));+ num_src1_rows++;+ }- // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses- if (ne0 % MATRIX_ROW_PADDING != 0) {- size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);- }+ if (num_src1_rows == 0) {+ continue;+ }- const char * buf_host = (const char *)data + offset_split;- CUDA_CHECK(cudaMemcpy(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice));- }-}+ src1_row.ne[1] = num_src1_rows;+ dst_row.ne[1] = num_src1_rows;-GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {- // split tensors must always be set in their entirety at once- GGML_ASSERT(offset == 0);- GGML_ASSERT(size == ggml_nbytes(tensor));+ src1_row.nb[1] = nb11;+ src1_row.nb[2] = num_src1_rows*nb11;+ src1_row.nb[3] = num_src1_rows*nb11;- ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;+ dst_row.nb[1] = nb1;+ dst_row.nb[2] = num_src1_rows*nb1;+ dst_row.nb[3] = num_src1_rows*nb1;- const int64_t ne0 = tensor->ne[0];- const size_t nb1 = tensor->nb[1];- ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;+ ggml_cuda_mul_mat(ctx, src0_row, &src1_row, &dst_row);- for (int id = 0; id < g_device_count; ++id) {- int64_t row_low, row_high;- get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);+ num_src1_rows = 0;+ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ const int32_t row_id_i = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);- int64_t nrows_split = row_high - row_low;- if (nrows_split == 0) {- continue;- }+ if (row_id_i != row_id) {+ continue;+ }- const size_t offset_split = row_low*nb1;- size_t size = ggml_nbytes_split(tensor, nrows_split);- const size_t original_size = size;+ GGML_ASSERT(row_id >= 0 && row_id < n_as);- // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses- if (ne0 % MATRIX_ROW_PADDING != 0) {- size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ CUDA_CHECK(cudaMemcpyAsync(dst_original + i01*nb1, dst_contiguous.get() + num_src1_rows*nb1,+ nb1, cudaMemcpyDeviceToDevice, stream));+ num_src1_rows++;+ }}-- char * buf_host = (char *)data + offset_split;- CUDA_CHECK(cudaMemcpy(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost));}}-GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {- UNUSED(buffer);- UNUSED(value);+static void ggml_cuda_scale(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_scale);}-static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {- /* .get_name = */ ggml_backend_cuda_split_buffer_get_name,- /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer,- /* .get_base = */ ggml_backend_cuda_split_buffer_get_base,- /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor,- /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor,- /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor,- /* .cpy_tensor = */ NULL,- /* .clear = */ ggml_backend_cuda_split_buffer_clear,- /* .reset = */ NULL,-};--// cuda split buffer type--GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) {- return GGML_CUDA_NAME "_Split";-- UNUSED(buft);+static void ggml_cuda_clamp(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_clamp);}-GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {- // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point- // instead, we allocate them for each tensor separately in init_tensor- // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,- // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct.- ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context();+static void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ const int64_t ne = ggml_nelements(src0);+ GGML_ASSERT(ne == ggml_nelements(src1));- return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);-}+ GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));-GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {- return 128;+ GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX);+ GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX);- UNUSED(buft);-}+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int64_t ne02 = src0->ne[2];-GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {- ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;+ //GGML_ASSERT(src0->ne[3] == 1);- size_t total_size = 0;+ const int64_t nb00 = src0->nb[0];+ const int64_t nb01 = src0->nb[1];+ const int64_t nb02 = src0->nb[2];+ const int64_t nb03 = src0->nb[3];- const int64_t ne0 = tensor->ne[0];+ const int64_t ne10 = src1->ne[0];+ const int64_t ne11 = src1->ne[1];+ const int64_t ne12 = src1->ne[2];- for (int id = 0; id < g_device_count; ++id) {- int64_t row_low, row_high;- get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id);+ //GGML_ASSERT(src1->ne[3] == 1);- int64_t nrows_split = row_high - row_low;- if (nrows_split == 0) {- continue;- }+ const int64_t nb10 = src1->nb[0];+ const int64_t nb11 = src1->nb[1];+ const int64_t nb12 = src1->nb[2];+ const int64_t nb13 = src1->nb[3];- total_size += ggml_nbytes_split(tensor, nrows_split);+ ggml_cuda_set_device(ctx.device);+ cudaStream_t main_stream = ctx.stream();- // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses- if (ne0 % MATRIX_ROW_PADDING != 0) {- total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);- }+ char * src0_ddc = (char *) src0->data;+ char * src1_ddc = (char *) src1->data;++ if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {+ ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {+ ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {+ ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {+ ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {+ ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {+ ggml_cpy_f32_q5_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {+ ggml_cpy_f32_iq4_nl_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {+ ggml_cpy_f32_q5_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {+ ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {+ ggml_cpy_f16_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else {+ fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,+ ggml_type_name(src0->type), ggml_type_name(src1->type));+ GGML_ASSERT(false);}- return total_size;+ GGML_UNUSED(dst);}-GGML_CALL static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {- return ggml_backend_is_cuda(backend);-- UNUSED(buft);+static void ggml_cuda_dup(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ // TODO: why do we pass dst as src1 here?+ ggml_cuda_cpy(ctx, src0, dst, nullptr);+ GGML_UNUSED(src1);}-GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {- return false;-- UNUSED(buft);+static void ggml_cuda_diag_mask_inf(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_diag_mask_inf);}-static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = {- /* .get_name = */ ggml_backend_cuda_split_buffer_type_name,- /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer,- /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,- /* .get_max_size = */ NULL, // defaults to SIZE_MAX- /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,- /* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend,- /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,-};--GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) {- // FIXME: this is not thread safe- static std::map<std::array<float, GGML_CUDA_MAX_DEVICES>, struct ggml_backend_buffer_type> buft_map;-- std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split_arr = {};-- bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; });- if (all_zero) {- tensor_split_arr = g_default_tensor_split;- } else {- float split_sum = 0.0f;- for (int i = 0; i < g_device_count; ++i) {- tensor_split_arr[i] = split_sum;- split_sum += tensor_split[i];- }- for (int i = 0; i < g_device_count; ++i) {- tensor_split_arr[i] /= split_sum;- }- }-- auto it = buft_map.find(tensor_split_arr);- if (it != buft_map.end()) {- return &it->second;- }-- struct ggml_backend_buffer_type buft {- /* .iface = */ ggml_backend_cuda_split_buffer_type_interface,- /* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr},- };-- auto result = buft_map.emplace(tensor_split_arr, buft);- return &result.first->second;+static void ggml_cuda_soft_max(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_soft_max);}-// host buffer type--GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) {- return GGML_CUDA_NAME "_Host";+static void ggml_cuda_rope(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_ASSERT(ggml_is_contiguous(src0)); // TODO: this restriction is temporary until non-cont support is implemented+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_rope);+}- UNUSED(buft);+static void ggml_cuda_alibi(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_alibi);}-GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) {- return GGML_CUDA_NAME "_Host";+static void ggml_cuda_pool2d(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_pool2d);+}- UNUSED(buffer);+static void ggml_cuda_im2col(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_im2col);}-GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {- ggml_cuda_host_free(buffer->context);+static void ggml_cuda_sum_rows(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_ASSERT(ggml_is_contiguous(src0));+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_sum_rows);}-GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {- void * ptr = ggml_cuda_host_malloc(size);+static void ggml_cuda_argsort(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_ASSERT(ggml_is_contiguous(src0));+ ggml_cuda_op_flatten(ctx, src0, src1, dst, ggml_cuda_op_argsort);+}- if (ptr == nullptr) {- // fallback to cpu buffer- return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);+static void ggml_cuda_nop(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_UNUSED(ctx);+ GGML_UNUSED(src0);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+}++static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * tensor) {+ // FIXME: where should this be?+ if (tensor->src[0] != nullptr && ggml_backend_buffer_is_cuda_split(tensor->src[0]->buffer)) {+ ggml_cuda_set_peer_access(tensor->src[1]->ne[1], ctx.device);}- ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);- buffer->buft = buft;- buffer->iface.get_name = ggml_backend_cuda_host_buffer_name;- buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;+ ggml_cuda_func_t func;- return buffer;+ switch (tensor->op) {+ case GGML_OP_REPEAT:+ func = ggml_cuda_repeat;+ break;+ case GGML_OP_GET_ROWS:+ func = ggml_cuda_get_rows;+ break;+ case GGML_OP_DUP:+ func = ggml_cuda_dup;+ break;+ case GGML_OP_ADD:+ func = ggml_cuda_add;+ break;+ case GGML_OP_ACC:+ func = ggml_cuda_acc;+ break;+ case GGML_OP_MUL:+ func = ggml_cuda_mul;+ break;+ case GGML_OP_DIV:+ func = ggml_cuda_div;+ break;+ case GGML_OP_UNARY:+ switch (ggml_get_unary_op(tensor)) {+ case GGML_UNARY_OP_GELU:+ func = ggml_cuda_gelu;+ break;+ case GGML_UNARY_OP_SILU:+ func = ggml_cuda_silu;+ break;+ case GGML_UNARY_OP_GELU_QUICK:+ func = ggml_cuda_gelu_quick;+ break;+ case GGML_UNARY_OP_TANH:+ func = ggml_cuda_tanh;+ break;+ case GGML_UNARY_OP_RELU:+ func = ggml_cuda_relu;+ break;+ case GGML_UNARY_OP_HARDSIGMOID:+ func = ggml_cuda_hardsigmoid;+ break;+ case GGML_UNARY_OP_HARDSWISH:+ func = ggml_cuda_hardswish;+ break;+ default:+ return false;+ }+ break;+ case GGML_OP_NORM:+ func = ggml_cuda_norm;+ break;+ case GGML_OP_GROUP_NORM:+ func = ggml_cuda_group_norm;+ break;+ case GGML_OP_CONCAT:+ func = ggml_cuda_concat;+ break;+ case GGML_OP_UPSCALE:+ func = ggml_cuda_upscale;+ break;+ case GGML_OP_PAD:+ func = ggml_cuda_pad;+ break;+ case GGML_OP_ARANGE:+ func = ggml_cuda_arange;+ break;+ case GGML_OP_TIMESTEP_EMBEDDING:+ func = ggml_cuda_timestep_embedding;+ break;+ case GGML_OP_LEAKY_RELU:+ func = ggml_cuda_leaky_relu;+ break;+ case GGML_OP_RMS_NORM:+ func = ggml_cuda_rms_norm;+ break;+ case GGML_OP_MUL_MAT:+ if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) {+ fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]);+ return false;+ } else {+ func = ggml_cuda_mul_mat;+ }+ break;+ case GGML_OP_MUL_MAT_ID:+ func = ggml_cuda_mul_mat_id;+ break;+ case GGML_OP_SCALE:+ func = ggml_cuda_scale;+ break;+ case GGML_OP_SQR:+ func = ggml_cuda_sqr;+ break;+ case GGML_OP_CLAMP:+ func = ggml_cuda_clamp;+ break;+ case GGML_OP_CPY:+ func = ggml_cuda_cpy;+ break;+ case GGML_OP_CONT:+ func = ggml_cuda_dup;+ break;+ case GGML_OP_NONE:+ case GGML_OP_RESHAPE:+ case GGML_OP_VIEW:+ case GGML_OP_PERMUTE:+ case GGML_OP_TRANSPOSE:+ func = ggml_cuda_nop;+ break;+ case GGML_OP_DIAG_MASK_INF:+ func = ggml_cuda_diag_mask_inf;+ break;+ case GGML_OP_SOFT_MAX:+ func = ggml_cuda_soft_max;+ break;+ case GGML_OP_ROPE:+ func = ggml_cuda_rope;+ break;+ case GGML_OP_ALIBI:+ func = ggml_cuda_alibi;+ break;+ case GGML_OP_IM2COL:+ func = ggml_cuda_im2col;+ break;+ case GGML_OP_POOL_2D:+ func = ggml_cuda_pool2d;+ break;+ case GGML_OP_SUM_ROWS:+ func = ggml_cuda_sum_rows;+ break;+ case GGML_OP_ARGSORT:+ func = ggml_cuda_argsort;+ break;+ default:+ return false;+ }++ func(ctx, tensor->src[0], tensor->src[1], tensor);+ return true;}-GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {- static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {- /* .iface = */ {- /* .get_name = */ ggml_backend_cuda_host_buffer_type_name,- /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,- /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,- /* .get_max_size = */ NULL, // defaults to SIZE_MAX- /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,- /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend,- /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,- },- /* .context = */ nullptr,- };- return &ggml_backend_cuda_buffer_type_host;-}+////////////////////////////////////////////////////////////////////////////////+// backend@@ -12224,49 +11456,84 @@ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_bufferGGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;- GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");- GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU);+ GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");- CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, g_cudaStreams[cuda_ctx->device][0]));+ CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));}GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;- GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");- GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU);+ GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");- CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0]));+ CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));}-GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) {- ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;+GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {+ GGML_ASSERT(ggml_backend_is_cuda(backend_src) || ggml_backend_is_cuda(backend_dst));- if (dst->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && ggml_backend_buffer_is_cuda(src->buffer)) {- CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx->device][0]));- return true;+ ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;+ ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;++ if (!ggml_backend_buffer_is_cuda(src->buffer)) {+ return false;}- return false;+ if (!ggml_backend_buffer_is_cuda(dst->buffer)) {+ return false;+ }++ // device -> device+ ggml_backend_cuda_context * cuda_ctx_src = (ggml_backend_cuda_context *)backend_src->context;+ ggml_backend_cuda_context * cuda_ctx_dst = (ggml_backend_cuda_context *)backend_dst->context;++ if (backend_src != backend_dst) {+ ggml_backend_cuda_buffer_context * buf_ctx_src = (ggml_backend_cuda_buffer_context *)buf_src->context;+ ggml_backend_cuda_buffer_context * buf_ctx_dst = (ggml_backend_cuda_buffer_context *)buf_dst->context;++ GGML_ASSERT(cuda_ctx_src->device == buf_ctx_src->device);+ GGML_ASSERT(cuda_ctx_dst->device == buf_ctx_dst->device);++ if (!cuda_ctx_src->copy_event) {+ ggml_cuda_set_device(cuda_ctx_src->device);+ CUDA_CHECK(cudaEventCreateWithFlags(&cuda_ctx_src->copy_event, cudaEventDisableTiming));+ }++ // copy on src stream+ if (cuda_ctx_src->device == cuda_ctx_dst->device) {+ CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream()));+ } else {+ CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream()));+ }++ // record event on src stream+ CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, cuda_ctx_src->stream()));++ // wait on dst stream for the copy to complete+ CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx_dst->stream(), cuda_ctx_src->copy_event, 0));+ } else {+ // src and dst are on the same backend+ CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream()));+ }+ return true;}GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;- CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[cuda_ctx->device][0]));+ CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));- UNUSED(backend);+ GGML_UNUSED(backend);}-GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {+GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;- ggml_cuda_set_main_device(cuda_ctx->device);+ ggml_cuda_set_device(cuda_ctx->device);- ggml_compute_params params = {};- params.type = GGML_TASK_TYPE_COMPUTE;- params.ith = 0;for (int i = 0; i < cgraph->n_nodes; i++) {ggml_tensor * node = cgraph->nodes[i];@@ -12275,27 +11542,22 @@ GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, gg}#ifndef NDEBUG- assert(node->backend == GGML_BACKEND_TYPE_GPU || node->backend == GGML_BACKEND_TYPE_GPU_SPLIT);assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));- assert(node->extra != nullptr);-for (int j = 0; j < GGML_MAX_SRC; j++) {if (node->src[j] != nullptr) {- assert(node->src[j]->backend == GGML_BACKEND_TYPE_GPU || node->src[j]->backend == GGML_BACKEND_TYPE_GPU_SPLIT);assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer));- assert(node->src[j]->extra != nullptr);}}#endif- bool ok = ggml_cuda_compute_forward(¶ms, node);+ bool ok = ggml_cuda_compute_forward(*cuda_ctx, node);if (!ok) {fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));}GGML_ASSERT(ok);}- return true;+ return GGML_STATUS_SUCCESS;}GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) {@@ -12373,6 +11635,15 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, consif (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) {return true;}+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) {+ return true;+ }+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) {+ return true;+ }+ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) {+ return true;+ }if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {return true;}@@ -12414,13 +11685,72 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, conscase GGML_OP_GROUP_NORM:case GGML_OP_UPSCALE:case GGML_OP_PAD:+ case GGML_OP_ARANGE:+ case GGML_OP_TIMESTEP_EMBEDDING:case GGML_OP_LEAKY_RELU:return true;default:return false;}- UNUSED(backend);+ GGML_UNUSED(backend);+}++GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const ggml_tensor * op) {+ const int min_batch_size = 32;++ return op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS;++ GGML_UNUSED(backend);+}++GGML_CALL static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;++ ggml_cuda_set_device(cuda_ctx->device);++ cudaEvent_t event;+ CUDA_CHECK(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));++ return new ggml_backend_event {+ /* .backend = */ backend,+ /* .context = */ event,+ };+}++GGML_CALL static void ggml_backend_cuda_event_free(ggml_backend_event_t event) {+ CUDA_CHECK(cudaEventDestroy((cudaEvent_t)event->context));++ delete event;+}++GGML_CALL static void ggml_backend_cuda_event_record(ggml_backend_event_t event) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)event->backend->context;++ CUDA_CHECK(cudaEventRecord((cudaEvent_t)event->context, cuda_ctx->stream()));+}++GGML_CALL static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;++ if (ggml_backend_is_cuda(event->backend)) {+ CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), (cudaEvent_t)event->context, 0));+ } else {+#if 0+ // untested+ auto wait_fn = [](void * user_data) {+ ggml_backend_event_t event = (ggml_backend_event_t)user_data;+ ggml_backend_event_synchronize(event);+ };++ CUDA_CHECK(cudaLaunchHostFunc(cuda_ctx->stream(), wait_fn, event));+#endif+ GGML_ASSERT(false);+ }+}++GGML_CALL static void ggml_backend_cuda_event_synchronize(ggml_backend_event_t event) {+ CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));}static ggml_backend_i ggml_backend_cuda_interface = {@@ -12436,25 +11766,33 @@ static ggml_backend_i ggml_backend_cuda_interface = {/* .graph_plan_compute = */ NULL,/* .graph_compute = */ ggml_backend_cuda_graph_compute,/* .supports_op = */ ggml_backend_cuda_supports_op,+ /* .offload_op = */ ggml_backend_cuda_offload_op,+ /* .event_new = */ ggml_backend_cuda_event_new,+ /* .event_free = */ ggml_backend_cuda_event_free,+ /* .event_record = */ ggml_backend_cuda_event_record,+ /* .event_wait = */ ggml_backend_cuda_event_wait,+ /* .event_synchronize = */ ggml_backend_cuda_event_synchronize,};-GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {- ggml_init_cublas(); // TODO: remove from ggml.c+static ggml_guid_t ggml_backend_cuda_guid() {+ static ggml_guid guid = { 0x2c, 0xdd, 0xe8, 0x1c, 0x65, 0xb3, 0x65, 0x73, 0x6a, 0x12, 0x88, 0x61, 0x1c, 0xc9, 0xdc, 0x25 };+ return &guid;+}- if (device < 0 || device >= ggml_cuda_get_device_count()) {+GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {+ if (device < 0 || device >= ggml_backend_cuda_get_device_count()) {fprintf(stderr, "%s: error: invalid device %d\n", __func__, device);return nullptr;}- // not strictly necessary, but it may reduce the overhead of the first graph_compute- ggml_cuda_set_main_device(device);-- ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context {- /* .device = */ device,- /* .name = */ GGML_CUDA_NAME + std::to_string(device),- };+ ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device);+ if (ctx == nullptr) {+ fprintf(stderr, "%s: error: failed to allocate context\n", __func__);+ return nullptr;+ }ggml_backend_t cuda_backend = new ggml_backend {+ /* .guid = */ ggml_backend_cuda_guid(),/* .interface = */ ggml_backend_cuda_interface,/* .context = */ ctx};@@ -12462,16 +11800,22 @@ GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {return cuda_backend;}+bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {+ return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;+}+GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend) {- return backend && backend->iface.get_name == ggml_backend_cuda_name;+ return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cuda_guid());}GGML_CALL int ggml_backend_cuda_get_device_count() {- return ggml_cuda_get_device_count();+ return get_cuda_global_info().device_count;}GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) {- ggml_cuda_get_device_description(device, description, description_size);+ cudaDeviceProp prop;+ CUDA_CHECK(cudaGetDeviceProperties(&prop, device));+ snprintf(description, description_size, "%s", prop.name);}GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) {@@ -12480,18 +11824,47 @@ GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, siCUDA_CHECK(cudaMemGetInfo(free, total));}+GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {+ if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {+ return false;+ }++ cudaError_t err = cudaHostRegister(buffer, size, cudaHostRegisterPortable | cudaHostRegisterReadOnly);+ if (err != cudaSuccess) {+ // clear the error+ cudaGetLastError();++ fprintf(stderr, "%s: warning: failed to register %.2f MiB of pinned memory: %s\n", __func__,+ size/1024.0/1024.0, cudaGetErrorString(err));+ return false;+ }+ return true;+}++GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer) {+ if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {+ return;+ }++ cudaError_t err = cudaHostUnregister(buffer);+ if (err != cudaSuccess) {+ // clear the error+ cudaGetLastError();+ }+}+// backend registryGGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data);return cuda_backend;- UNUSED(params);+ GGML_UNUSED(params);}extern "C" GGML_API GGML_CALL int ggml_backend_cuda_reg_devices();GGML_API GGML_CALL int ggml_backend_cuda_reg_devices() {- int device_count = ggml_cuda_get_device_count();+ int device_count = ggml_backend_cuda_get_device_count();//int device_count = 1; // DEBUG: some tools require delaying CUDA initializationfor (int i = 0; i < device_count; i++) {char name[128];diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.hindex 37cb0d5b..c8f94ea8 100644--- a/llama.cpp/ggml-cuda.h+++ b/llama.cpp/ggml-cuda.h@@ -17,31 +17,19 @@ extern "C" {#define GGML_CUDA_MAX_DEVICES 16-GGML_API GGML_CALL void ggml_cuda_link(const struct ggml_backend_api * backend_api);--// Always success. To check if CUDA is actually loaded, use `ggml_cublas_loaded`.-GGML_API GGML_CALL void ggml_init_cublas(void);--// Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`.-GGML_API GGML_CALL bool ggml_cublas_loaded(void);--GGML_API GGML_CALL void * ggml_cuda_host_malloc(size_t size);-GGML_API GGML_CALL void ggml_cuda_host_free(void * ptr);--GGML_API GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);-GGML_API GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);--GGML_API GGML_CALL int ggml_cuda_get_device_count(void);-GGML_API GGML_CALL void ggml_cuda_get_device_description(int device, char * description, size_t description_size);+GGML_API GGML_CALL bool ggml_cuda_link(const struct ggml_backend_api * backend_api);// backend APIGGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device);GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend);+// device bufferGGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);+// split tensor buffer that splits matrices by rows across multiple devicesGGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split);+// pinned host buffer for use with the CPU backend for faster copies between CPU and GPUGGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);@@ -49,6 +37,9 @@ GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void);GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);+GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);+GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer);+#ifdef __cplusplus}#endifdiff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex fe35cc4c..324b7217 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -53,26 +53,30 @@ extern "C" {//#include <arm_neon.h>+typedef __fp16 ggml_fp16_internal_t;+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)#define GGML_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {- __fp16 tmp;+ ggml_fp16_internal_t tmp;memcpy(&tmp, &h, sizeof(ggml_fp16_t));return (float)tmp;}static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {ggml_fp16_t res;- __fp16 tmp = f;+ ggml_fp16_internal_t tmp = f;memcpy(&res, &tmp, sizeof(ggml_fp16_t));return res;}#else+typedef uint16_t ggml_fp16_internal_t;+#ifdef __wasm_simd128__#include <wasm_simd128.h>#elsediff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex 4ad7545c..81d19be2 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -188,6 +188,8 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_IM2COL_F32,GGML_METAL_KERNEL_TYPE_UPSCALE_F32,GGML_METAL_KERNEL_TYPE_PAD_F32,+ GGML_METAL_KERNEL_TYPE_ARANGE_F32,+ GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32,GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC,GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32,@@ -196,8 +198,9 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0,GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0,GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1,- //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0,- //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1,+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0,+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1,+ GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL,GGML_METAL_KERNEL_TYPE_CPY_F16_F16,GGML_METAL_KERNEL_TYPE_CPY_F16_F32,GGML_METAL_KERNEL_TYPE_CONCAT,@@ -303,6 +306,11 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {id<MTLLibrary> metal_library;// load library+ //+ // - first check if the library is embedded+ // - then check if the library is in the bundle+ // - if not found, load the source and compile it+ // - if that fails, return NULL{NSBundle * bundle = nil;#ifdef SWIFT_PACKAGE@@ -310,12 +318,21 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {#elsebundle = [NSBundle bundleForClass:[GGMLMetalClass class]];#endif+NSError * error = nil;- NSString * libPath = [bundle pathForResource:@"default" ofType:@"metallib"];- if (libPath != nil) {++#if GGML_METAL_EMBED_LIBRARY+ const bool try_metallib = false;+#else+ const bool try_metallib = true;+#endif++ NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];+ if (try_metallib && path_lib != nil) {// pre-compiled library found- NSURL * libURL = [NSURL fileURLWithPath:libPath];- GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]);+ NSURL * libURL = [NSURL fileURLWithPath:path_lib];+ GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]);+metal_library = [ctx->device newLibraryWithURL:libURL error:&error];if (error) {GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);@@ -328,38 +345,41 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {extern const char ggml_metallib_start[];extern const char ggml_metallib_end[];- NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];+ NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];#elseGGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);- NSString * sourcePath;- NSString * ggmlMetalPathResources = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];+ NSString * path_source;+ NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];- GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, ggmlMetalPathResources ? [ggmlMetalPathResources UTF8String] : "nil");+ GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil");- if (ggmlMetalPathResources) {- sourcePath = [ggmlMetalPathResources stringByAppendingPathComponent:@"ggml-metal.metal"];+ if (path_resource) {+ path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"];} else {- sourcePath = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];+ path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];}- if (sourcePath == nil) {++ if (path_source == nil) {GGML_METAL_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__);- sourcePath = @"ggml-metal.metal";+ path_source = @"ggml-metal.metal";}- GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [sourcePath UTF8String]);- NSString * src = [NSString stringWithContentsOfFile:sourcePath encoding:NSUTF8StringEncoding error:&error];++ GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]);++ NSString * src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error];if (error) {GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);return NULL;}-#endif+#endif // GGML_METAL_EMBED_LIBRARY@autoreleasepool {// dictionary of preprocessor macrosNSMutableDictionary * prep = [NSMutableDictionary dictionary];#ifdef GGML_QKK_64- prep[@"QK_K"] = @(64);+ prep[@"GGML_QKK_64"] = @(1);#endifMTLCompileOptions* options = [MTLCompileOptions new];@@ -594,6 +614,8 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);@@ -602,8 +624,9 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);@@ -722,6 +745,8 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constreturn false;case GGML_OP_UPSCALE:case GGML_OP_PAD:+ case GGML_OP_ARANGE:+ case GGML_OP_TIMESTEP_EMBEDDING:case GGML_OP_ARGSORT:case GGML_OP_LEAKY_RELU:return true;@@ -741,6 +766,9 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constcase GGML_TYPE_Q8_0:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_IQ4_NL:return true;default:return false;@@ -767,7 +795,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const}}-static bool ggml_metal_graph_compute(+static enum ggml_status ggml_metal_graph_compute(struct ggml_metal_context * ctx,struct ggml_cgraph * gf) {@@ -1116,7 +1144,8 @@ static bool ggml_metal_graph_compute({GGML_ASSERT(ggml_is_contiguous(src0));- const float scale = *(const float *) dst->op_params;+ float scale;+ memcpy(&scale, dst->op_params, sizeof(scale));int64_t n = ggml_nelements(dst);@@ -1275,11 +1304,15 @@ static bool ggml_metal_graph_compute(pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline;}- const float scale = ((float *) dst->op_params)[0];- const float max_bias = ((float *) dst->op_params)[1];+ float scale;+ float max_bias;++ memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(scale));+ memcpy(&max_bias, ((int32_t *) dst->op_params) + 1, sizeof(max_bias));const int64_t nrows_x = ggml_nrows(src0);const int64_t nrows_y = src0->ne[1];+const uint32_t n_head_kv = nrows_x/nrows_y;const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv));@@ -1656,8 +1689,8 @@ static bool ggml_metal_graph_compute(// TODO: make this more generalGGML_ASSERT(n_as <= 8);- // max size of the src1ids array in the kernel stack- GGML_ASSERT(ne11 <= 512);+ // max size of the src1ids array in the kernel shared buffer+ GGML_ASSERT(ne11 <= 4096);const int64_t ne20 = src2 ? src2->ne[0] : 0;const int64_t ne21 = src2 ? src2->ne[1] : 0;@@ -1755,7 +1788,7 @@ static bool ggml_metal_graph_compute([encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j];}- [encoder setThreadgroupMemoryLength:8192 atIndex:0];+ [encoder setThreadgroupMemoryLength:GGML_PAD(8192 + 2*ne11, 16) atIndex:0];[encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];} else {@@ -2111,6 +2144,7 @@ static bool ggml_metal_graph_compute(//const int n_past = ((int32_t *) dst->op_params)[0];const int n_head = ((int32_t *) dst->op_params)[1];+float max_bias;memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));@@ -2325,6 +2359,50 @@ static bool ggml_metal_graph_compute([encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];} break;+ case GGML_OP_ARANGE:+ {+ GGML_ASSERT(dst->type == GGML_TYPE_F32);++ float start;+ float step;++ memcpy(&start, ((int32_t *) dst->op_params) + 0, sizeof(float));+ memcpy(&step, ((int32_t *) dst->op_params) + 2, sizeof(float));++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARANGE_F32].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:0];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:1];+ [encoder setBytes:&start length:sizeof(start) atIndex:2];+ [encoder setBytes:&step length:sizeof(step) atIndex:3];++ const int nth = MIN(1024, ne0);++ [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_TIMESTEP_EMBEDDING:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F32);++ const int dim = dst->op_params[0];+ const int max_period = dst->op_params[1];++ const int half = dim / 2;++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:2];+ [encoder setBytes:&dim length:sizeof(dim) atIndex:3];+ [encoder setBytes:&max_period length:sizeof(max_period) atIndex:4];++ const int nth = MIN(1024, half);++ [encoder dispatchThreadgroups:MTLSizeMake(ne00, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;case GGML_OP_ARGSORT:{GGML_ASSERT(src0->type == GGML_TYPE_F32);@@ -2383,13 +2461,14 @@ static bool ggml_metal_graph_compute(GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);switch (dstt) {- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break;- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break;- case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break;- case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break;- case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break;- //case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break;- //case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break;+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break;+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break;+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break;+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break;+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break;+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break;+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break;+ case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL].pipeline; break;default: GGML_ASSERT(false && "not implemented");};} break;@@ -2453,7 +2532,7 @@ static bool ggml_metal_graph_compute(MTLCommandBufferStatus status = [command_buffer status];if (status != MTLCommandBufferStatusCompleted) {GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);- return false;+ return GGML_STATUS_FAILED;}}@@ -2462,7 +2541,7 @@ static bool ggml_metal_graph_compute(}}- return true;+ return GGML_STATUS_SUCCESS;}////////////////////////////////////////////////////////////////////////////////@@ -2764,7 +2843,7 @@ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffeUNUSED(backend);}-GGML_CALL static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+GGML_CALL static enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;return ggml_metal_graph_compute(metal_ctx, cgraph);@@ -2789,6 +2868,12 @@ static struct ggml_backend_i ggml_backend_metal_i = {/* .graph_plan_compute = */ NULL,/* .graph_compute = */ ggml_backend_metal_graph_compute,/* .supports_op = */ ggml_backend_metal_supports_op,+ /* .offload_op = */ NULL,+ /* .event_new = */ NULL,+ /* .event_free = */ NULL,+ /* .event_record = */ NULL,+ /* .event_wait = */ NULL,+ /* .event_synchronize = */ NULL,};void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {@@ -2796,6 +2881,11 @@ void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void *ggml_metal_log_user_data = user_data;}+static ggml_guid_t ggml_backend_metal_guid(void) {+ static ggml_guid guid = { 0x81, 0xa1, 0x8b, 0x1e, 0x71, 0xec, 0x79, 0xed, 0x2b, 0x85, 0xdc, 0x8a, 0x61, 0x98, 0x30, 0xe6 };+ return &guid;+}+ggml_backend_t ggml_backend_metal_init(void) {struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);@@ -2806,6 +2896,7 @@ ggml_backend_t ggml_backend_metal_init(void) {ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend));*metal_backend = (struct ggml_backend) {+ /* .guid = */ ggml_backend_metal_guid(),/* .interface = */ ggml_backend_metal_i,/* .context = */ ctx,};@@ -2813,8 +2904,12 @@ ggml_backend_t ggml_backend_metal_init(void) {return metal_backend;}+bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {+ return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;+}+bool ggml_backend_is_metal(ggml_backend_t backend) {- return backend && backend->iface.get_name == ggml_backend_metal_name;+ return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_metal_guid());}void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metalindex 68941190..f97321f6 100644--- a/llama.cpp/ggml-metal.metal+++ b/llama.cpp/ggml-metal.metal@@ -1,3795 +1,696 @@+#define GGML_COMMON_DECL_METAL+#define GGML_COMMON_IMPL_METAL++#ifndef GGML_COMMON_DECL++#if defined(GGML_COMMON_DECL_C)+#include <stdint.h>++typedef uint16_t ggml_half;+typedef uint32_t ggml_half2;++#define GGML_COMMON_AGGR++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_METAL)#include <metal_stdlib>-using namespace metal;+typedef half ggml_half;+typedef half2 ggml_half2;-#define MAX(x, y) ((x) > (y) ? (x) : (y))-#define MIN(x, y) ((x) < (y) ? (x) : (y))-#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }+#define GGML_COMMON_AGGR-#define QK4_0 32+#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_CUDA)+#include <cuda_fp16.h>+#include <cstdint>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR data++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_HIP)+#include <hip/hip_fp16.h>+#include <cstdint>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR data++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_SYCL)+#include <sycl/half_type.hpp>+#include <cstdint>++typedef sycl::half ggml_half;+typedef sycl::half2 ggml_half2;++#define GGML_COMMON_AGGR data++#define GGML_COMMON_DECL+#endif++#if defined(GGML_COMMON_DECL)++#ifndef __cplusplus+#ifndef static_assert+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)+#define static_assert(cond, msg) _Static_assert(cond, msg)+#else+#define static_assert(cond, msg) struct global_scope_noop_trick+#endif+#endif+#endif // __cplusplus++// QK = number of values after dequantization+// QK_K = super-block size++#ifdef GGML_QKK_64+#define QK_K 64+#define K_SCALE_SIZE 4+#else+#define QK_K 256+#define K_SCALE_SIZE 12+#endif // GGML_QKK_64++#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL)+// QR = QK / number of values before dequantization+// QI = number of 32 bit integers before dequantization++#define QI4_0 (QK4_0 / (4 * QR4_0))#define QR4_0 2++#define QI4_1 (QK4_1 / (4 * QR4_1))+#define QR4_1 2++#define QI5_0 (QK5_0 / (4 * QR5_0))+#define QR5_0 2++#define QI5_1 (QK5_1 / (4 * QR5_1))+#define QR5_1 2++#define QI8_0 (QK8_0 / (4 * QR8_0))+#define QR8_0 1++#define QI8_1 (QK8_1 / (4 * QR8_1))+#define QR8_1 1++#define QI2_K (QK_K / (4*QR2_K))+#define QR2_K 4++#define QI3_K (QK_K / (4*QR3_K))+#define QR3_K 4++#define QI4_K (QK_K / (4*QR4_K))+#define QR4_K 2++#define QI5_K (QK_K / (4*QR5_K))+#define QR5_K 2++#define QI6_K (QK_K / (4*QR6_K))+#define QR6_K 2++#define QI2_XXS (QK_K / (4*QR2_XXS))+#define QR2_XXS 8++#define QI2_XS (QK_K / (4*QR2_XS))+#define QR2_XS 8++#define QI2_S (QK_K / (4*QR2_S))+#define QR2_S 8++#define QI3_XXS (QK_K / (4*QR3_XXS))+#define QR3_XXS 8++#define QI3_XS (QK_K / (4*QR3_XS))+#define QR3_XS 8++#define QI1_S (QK_K / (4*QR1_S))+#define QR1_S 8++#define QI4_NL (QK4_NL / (4*QR4_NL))+#define QR4_NL 2++#if QK_K == 64+#define QI4_XS QI4_NL+#define QR4_XS QR4_NL+#else+#define QI4_XS (QK_K / (4*QR4_XS))+#define QR4_XS 8+#endif++#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP++#define QK4_0 32typedef struct {- half d; // delta+ ggml_half d; // deltauint8_t qs[QK4_0 / 2]; // nibbles / quants} block_q4_0;+static_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, "wrong q4_0 block size/padding");#define QK4_1 32typedef struct {- half d; // delta- half m; // min- uint8_t qs[QK4_1 / 2]; // nibbles / quants+ union {+ struct {+ ggml_half d; // delta+ ggml_half m; // min+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t qs[QK4_1 / 2]; // nibbles / quants} block_q4_1;+static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");#define QK5_0 32typedef struct {- half d; // delta+ ggml_half d; // deltauint8_t qh[4]; // 5-th bit of quantsuint8_t qs[QK5_0 / 2]; // nibbles / quants} block_q5_0;+static_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");#define QK5_1 32typedef struct {- half d; // delta- half m; // min- uint8_t qh[4]; // 5-th bit of quants- uint8_t qs[QK5_1 / 2]; // nibbles / quants+ union {+ struct {+ ggml_half d; // delta+ ggml_half m; // min+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t qh[4]; // 5-th bit of quants+ uint8_t qs[QK5_1 / 2]; // nibbles / quants} block_q5_1;+static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");#define QK8_0 32typedef struct {- half d; // delta+ ggml_half d; // deltaint8_t qs[QK8_0]; // quants} block_q8_0;+static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding");-#define N_SIMDWIDTH 32 // assuming SIMD group size is 32+#define QK8_1 32+typedef struct {+ union {+ struct {+ ggml_half d; // delta+ ggml_half s; // d * sum(qs[i])+ } GGML_COMMON_AGGR;+ ggml_half2 ds;+ };+ int8_t qs[QK8_1]; // quants+} block_q8_1;+static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding");-enum ggml_sort_order {- GGML_SORT_ASC,- GGML_SORT_DESC,-};+//+// Super-block quantization structures+//-// general-purpose kernel for addition, multiplication and division of two tensors-// pros: works for non-contiguous tensors, supports broadcast across all dims-// cons: not very efficient-kernel void kernel_add(- device const char * src0,- device const char * src1,- device char * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant int64_t & ne13,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant uint64_t & nb13,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- constant int64_t & offs,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig.z;- const int64_t i02 = tgpig.y;- const int64_t i01 = tgpig.x;+// 2-bit quantization+// weight is represented as x = a * q + b+// 16 blocks of 16 elements each+// Effectively 2.625 bits per weight+typedef struct {+ uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits+ uint8_t qs[QK_K/4]; // quants+ union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+} block_q2_K;+static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");- const int64_t i13 = i03 % ne13;- const int64_t i12 = i02 % ne12;- const int64_t i11 = i01 % ne11;+// 3-bit quantization+// weight is represented as x = a * q+// 16 blocks of 16 elements each+// Effectively 3.4375 bits per weight+#ifdef GGML_QKK_64+typedef struct {+ uint8_t hmask[QK_K/8]; // quants - high bit+ uint8_t qs[QK_K/4]; // quants - low 2 bits+ uint8_t scales[2];+ ggml_half d; // super-block scale+} block_q3_K;+static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding");+#else+typedef struct {+ uint8_t hmask[QK_K/8]; // quants - high bit+ uint8_t qs[QK_K/4]; // quants - low 2 bits+ uint8_t scales[12]; // scales, quantized with 6 bits+ ggml_half d; // super-block scale+} block_q3_K;+static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");+#endif- device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + offs;- device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;- device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + offs;+// 4-bit quantization+// 8 blocks of 32 elements each+// weight is represented as x = a * q + b+// Effectively 4.5 bits per weight+#ifdef GGML_QKK_64+typedef struct {+ ggml_half d[2]; // super-block scales/mins+ uint8_t scales[2]; // 4-bit block scales/mins+ uint8_t qs[QK_K/2]; // 4--bit quants+} block_q4_K;+static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + QK_K/2 + 2, "wrong q4_K block size/padding");+#else+typedef struct {+ union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits+ uint8_t qs[QK_K/2]; // 4--bit quants+} block_q4_K;+static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");+#endif- for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- const int i10 = i0 % ne10;- *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) + *((device float *)(src1_ptr + i10*nb10));- }-}+// 5-bit quantization+// 8 blocks of 32 elements each+// weight is represented as x = a * q + b+// Effectively 5.5 bits per weight+#ifdef GGML_QKK_64+typedef struct {+ ggml_half d; // super-block scale+ int8_t scales[QK_K/16]; // 8-bit block scales+ uint8_t qh[QK_K/8]; // quants, high bit+ uint8_t qs[QK_K/2]; // quants, low 4 bits+} block_q5_K;+static_assert(sizeof(block_q5_K) == sizeof(ggml_half) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding");+#else+typedef struct {+ union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR;+ ggml_half2 dm;+ };+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits+ uint8_t qh[QK_K/8]; // quants, high bit+ uint8_t qs[QK_K/2]; // quants, low 4 bits+} block_q5_K;+static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");+#endif-kernel void kernel_mul(- device const char * src0,- device const char * src1,- device char * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant int64_t & ne13,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant uint64_t & nb13,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig.z;- const int64_t i02 = tgpig.y;- const int64_t i01 = tgpig.x;+// 6-bit quantization+// weight is represented as x = a * q+// 16 blocks of 16 elements each+// Effectively 6.5625 bits per weight+typedef struct {+ uint8_t ql[QK_K/2]; // quants, lower 4 bits+ uint8_t qh[QK_K/4]; // quants, upper 2 bits+ int8_t scales[QK_K/16]; // scales, quantized with 8 bits+ ggml_half d; // super-block scale+} block_q6_K;+static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");- const int64_t i13 = i03 % ne13;- const int64_t i12 = i02 % ne12;- const int64_t i11 = i01 % ne11;+// This is only used for intermediate quantization and dot products+typedef struct {+ float d; // delta+ int8_t qs[QK_K]; // quants+ int16_t bsums[QK_K/16]; // sum of quants in groups of 16+} block_q8_K;+static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");++// (Almost) "true" 2-bit quantization.+// Due to the need to use blocks as per ggml design, it ends up using+// 2.0625 bpw because of the 16-bit scale for each block of 256.+typedef struct {+ ggml_half d;+ uint16_t qs[QK_K/8];+} block_iq2_xxs;+static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");- device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;- device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;- device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;+// 2.3125 bpw quants+typedef struct {+ ggml_half d;+ uint16_t qs[QK_K/8];+ uint8_t scales[QK_K/32];+} block_iq2_xs;+static_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");- for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- const int i10 = i0 % ne10;- *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) * *((device float *)(src1_ptr + i10*nb10));- }-}+// 2.5625 bpw quants+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/4];+ uint8_t qh[QK_K/32];+ uint8_t scales[QK_K/32];+} block_iq2_s;+static_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding");-kernel void kernel_div(- device const char * src0,- device const char * src1,- device char * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant int64_t & ne13,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant uint64_t & nb13,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig.z;- const int64_t i02 = tgpig.y;- const int64_t i01 = tgpig.x;+// (Almost) "true" 3-bit quantization.+// Due to the need to use blocks as per ggml design, it ends up using+// 3.0625 bpw because of the 16-bit scale for each block of 256.+typedef struct {+ ggml_half d;+ uint8_t qs[3*QK_K/8];+} block_iq3_xxs;+static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");- const int64_t i13 = i03 % ne13;- const int64_t i12 = i02 % ne12;- const int64_t i11 = i01 % ne11;+// 3.4375 bpw+#if QK_K == 64+#define IQ3S_N_SCALE 2+#else+#define IQ3S_N_SCALE QK_K/64+#endif+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/4];+ uint8_t qh[QK_K/32];+ uint8_t signs[QK_K/8];+ uint8_t scales[IQ3S_N_SCALE];+} block_iq3_s;+static_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding");- device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;- device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;- device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;-- for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- const int i10 = i0 % ne10;- *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) / *((device float *)(src1_ptr + i10*nb10));- }-}--// assumption: src1 is a row-// broadcast src1 into src0-kernel void kernel_add_row(- device const float4 * src0,- device const float4 * src1,- device float4 * dst,- constant uint64_t & nb [[buffer(28)]],- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] + src1[tpig % nb];-}--kernel void kernel_mul_row(- device const float4 * src0,- device const float4 * src1,- device float4 * dst,- constant uint64_t & nb [[buffer(28)]],- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] * src1[tpig % nb];-}--kernel void kernel_div_row(- device const float4 * src0,- device const float4 * src1,- device float4 * dst,- constant uint64_t & nb [[buffer(28)]],- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] / src1[tpig % nb];-}--kernel void kernel_scale(- device const float * src0,- device float * dst,- constant float & scale,- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] * scale;-}+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/8];+ uint16_t qh[QK_K/32];+} block_iq1_s;+static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");-kernel void kernel_scale_4(- device const float4 * src0,- device float4 * dst,- constant float & scale,- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] * scale;-}+// Non-linear quants+#define QK4_NL 32+typedef struct {+ ggml_half d;+ uint8_t qs[QK4_NL/2];+} block_iq4_nl;+static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding");-kernel void kernel_relu(- device const float * src0,- device float * dst,- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = max(0.0f, src0[tpig]);-}+#if QK_K == 64+#define block_iq4_xs block_iq4_nl+#else+typedef struct {+ ggml_half d;+ uint16_t scales_h;+ uint8_t scales_l[QK_K/64];+ uint8_t qs[QK_K/2];+} block_iq4_xs;+static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");+#endif-kernel void kernel_tanh(- device const float * src0,- device float * dst,- uint tpig[[thread_position_in_grid]]) {- device const float & x = src0[tpig];- dst[tpig] = precise::tanh(x);-}+#endif // GGML_COMMON_DECL+#endif // GGML_COMMON_DECL-constant float GELU_COEF_A = 0.044715f;-constant float GELU_QUICK_COEF = -1.702f;-constant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;+////////////////////////////////////////////////////////////////////////////////-kernel void kernel_gelu(- device const float4 * src0,- device float4 * dst,- uint tpig[[thread_position_in_grid]]) {- device const float4 & x = src0[tpig];+#ifndef GGML_COMMON_IMPL- // BEWARE !!!- // Simply using "tanh" instead of "precise::tanh" will sometimes results in NaNs!- // This was observed with Falcon 7B and 40B models- //- dst[tpig] = 0.5f*x*(1.0f + precise::tanh(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));-}+#if defined(GGML_COMMON_IMPL_C)+#include <stdint.h>-kernel void kernel_gelu_quick(- device const float4 * src0,- device float4 * dst,- uint tpig[[thread_position_in_grid]]) {- device const float4 & x = src0[tpig];+#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {+#define GGML_TABLE_END() };- dst[tpig] = x*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x)));-}+#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_METAL)+#include <metal_stdlib>-kernel void kernel_silu(- device const float4 * src0,- device float4 * dst,- uint tpig[[thread_position_in_grid]]) {- device const float4 & x = src0[tpig];- dst[tpig] = x / (1.0f + exp(-x));-}+#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = {+#define GGML_TABLE_END() };-kernel void kernel_sqr(- device const float * src0,- device float * dst,- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] * src0[tpig];-}+#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP)+#include <cstdint>-kernel void kernel_sum_rows(- device const float * src0,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant int64_t & ne13,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant uint64_t & nb13,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tpig[[thread_position_in_grid]]) {- int64_t i3 = tpig.z;- int64_t i2 = tpig.y;- int64_t i1 = tpig.x;+#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = {+#define GGML_TABLE_END() };- if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {- return;- }+#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_SYCL)+#include <cstdint>- device const float * src_row = (device const float *) ((device const char *) src0 + i1*nb01 + i2*nb02 + i3*nb03);- device float * dst_row = (device float *) ((device char *) dst + i1*nb1 + i2*nb2 + i3*nb3);+#define GGML_TABLE_BEGIN(type, name, size) static dpct::global_memory<const type, 1> name(sycl::range<1>(size), {+#define GGML_TABLE_END() });- float row_sum = 0;+#define GGML_COMMON_IMPL+#endif- for (int64_t i0 = 0; i0 < ne00; i0++) {- row_sum += src_row[i0];- }+#if defined(GGML_COMMON_IMPL)- dst_row[0] = row_sum;-}+GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)+ 1, 2, 4, 8, 16, 32, 64, 128+GGML_TABLE_END()-kernel void kernel_soft_max(- device const float * src0,- device const float * src1,- device const float * src2,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant float & scale,- constant float & max_bias,- constant float & m0,- constant float & m1,- constant uint32_t & n_head_log2,- threadgroup float * buf [[threadgroup(0)]],- uint tgpig[[threadgroup_position_in_grid]],- uint tpitg[[thread_position_in_threadgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]],- uint tiisg[[thread_index_in_simdgroup]],- uint ntg[[threads_per_threadgroup]]) {- const int64_t i03 = (tgpig) / (ne02*ne01);- const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;- const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);+GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)+ 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,+ 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,+ 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,+ 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,+ 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,+ 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,+ 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,+ 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,+GGML_TABLE_END()++//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)+ 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,+ 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,+ 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,+ 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,+ 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,+ 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,+ 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,+ 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,+ 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,+ 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,+ 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,+ 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,+ 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,+ 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,+ 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,+ 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,+ 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,+ 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,+ 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,+ 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,+ 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,+ 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,+ 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,+ 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,+ 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,+ 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,+ 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,+ 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,+ 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,+ 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,+ 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,+ 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,+GGML_TABLE_END()+//#endif+++GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,+ 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,+ 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,+ 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,+ 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,+ 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,+ 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,+ 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,+ 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,+ 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,+ 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,+ 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,+ 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,+ 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,+ 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,+ 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,+ 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,+ 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,+ 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,+ 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,+ 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,+ 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,+ 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,+ 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,+ 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,+ 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,+ 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,+ 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,+ 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,+ 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,+ 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,+ 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,+ 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,+ 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,+ 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,+ 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,+ 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,+ 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,+ 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,+ 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,+ 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,+ 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,+ 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,+ 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,+ 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,+ 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,+ 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,+ 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,+ 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,+ 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,+ 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,+ 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,+ 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,+ 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,+ 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,+ 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,+ 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,+ 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,+ 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,+ 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,+GGML_TABLE_END()- device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;- device const float * pmask = src1 != src0 ? src1 + i01*ne00 : nullptr;- device const float * ppos = src2 != src0 ? src2 : nullptr;- device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- float slope = 0.0f;-- // ALiBi- if (max_bias > 0.0f) {- const int64_t h = i02;-- const float base = h < n_head_log2 ? m0 : m1;- const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;-- slope = pow(base, exp);- }-- // parallel max- float lmax = -INFINITY;-- for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));- }-- // find the max value in the block- float max_val = simd_max(lmax);- if (ntg > N_SIMDWIDTH) {- if (sgitg == 0) {- buf[tiisg] = -INFINITY;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (tiisg == 0) {- buf[sgitg] = max_val;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- max_val = buf[tiisg];- max_val = simd_max(max_val);- }-- // parallel sum- float lsum = 0.0f;- for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);- lsum += exp_psrc0;- pdst[i00] = exp_psrc0;- }-- // This barrier fixes a failing test- // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335- threadgroup_barrier(mem_flags::mem_none);-- float sum = simd_sum(lsum);-- if (ntg > N_SIMDWIDTH) {- if (sgitg == 0) {- buf[tiisg] = 0.0f;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (tiisg == 0) {- buf[sgitg] = sum;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- sum = buf[tiisg];- sum = simd_sum(sum);- }-- const float inv_sum = 1.0f/sum;-- for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- pdst[i00] *= inv_sum;- }-}--kernel void kernel_soft_max_4(- device const float * src0,- device const float * src1,- device const float * src2,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant float & scale,- constant float & max_bias,- constant float & m0,- constant float & m1,- constant uint32_t & n_head_log2,- threadgroup float * buf [[threadgroup(0)]],- uint tgpig[[threadgroup_position_in_grid]],- uint tpitg[[thread_position_in_threadgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]],- uint tiisg[[thread_index_in_simdgroup]],- uint ntg[[threads_per_threadgroup]]) {- const int64_t i03 = (tgpig) / (ne02*ne01);- const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;- const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);-- device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);- device const float4 * pmask = src1 != src0 ? (device const float4 *)(src1 + i01*ne00) : nullptr;- device const float4 * ppos = src2 != src0 ? (device const float4 *)(src2) : nullptr;- device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);-- float slope = 0.0f;-- if (max_bias > 0.0f) {- const int64_t h = i02;-- const float base = h < n_head_log2 ? m0 : m1;- const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;-- slope = pow(base, exp);- }-- // parallel max- float4 lmax4 = -INFINITY;-- for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));- }-- const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));+GGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,+GGML_TABLE_END()- float max_val = simd_max(lmax);- if (ntg > N_SIMDWIDTH) {- if (sgitg == 0) {- buf[tiisg] = -INFINITY;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (tiisg == 0) {- buf[sgitg] = max_val;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- max_val = buf[tiisg];- max_val = simd_max(max_val);- }-- // parallel sum- float4 lsum4 = 0.0f;- for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);- lsum4 += exp_psrc4;- pdst4[i00] = exp_psrc4;- }-- const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];-- // This barrier fixes a failing test- // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335- threadgroup_barrier(mem_flags::mem_none);-- float sum = simd_sum(lsum);-- if (ntg > N_SIMDWIDTH) {- if (sgitg == 0) {- buf[tiisg] = 0.0f;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (tiisg == 0) {- buf[sgitg] = sum;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- sum = buf[tiisg];- sum = simd_sum(sum);- }-- const float inv_sum = 1.0f/sum;-- for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- pdst4[i00] *= inv_sum;- }-}--kernel void kernel_diag_mask_inf(- device const float * src0,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int & n_past,- uint3 tpig[[thread_position_in_grid]]) {- const int64_t i02 = tpig[2];- const int64_t i01 = tpig[1];- const int64_t i00 = tpig[0];-- if (i00 > n_past + i01) {- dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY;- } else {- dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00];- }-}--kernel void kernel_diag_mask_inf_8(- device const float4 * src0,- device float4 * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int & n_past,- uint3 tpig[[thread_position_in_grid]]) {-- const int64_t i = 2*tpig[0];-- dst[i+0] = src0[i+0];- dst[i+1] = src0[i+1];- int64_t i4 = 4*i;- const int64_t i02 = i4/(ne00*ne01); i4 -= i02*ne00*ne01;- const int64_t i01 = i4/(ne00); i4 -= i01*ne00;- const int64_t i00 = i4;- for (int k = 3; k >= 0; --k) {- if (i00 + 4 + k <= n_past + i01) {- break;- }- dst[i+1][k] = -INFINITY;- if (i00 + k > n_past + i01) {- dst[i][k] = -INFINITY;- }- }-}--kernel void kernel_norm(- device const void * src0,- device float * dst,- constant int64_t & ne00,- constant uint64_t & nb01,- constant float & eps,- threadgroup float * sum [[threadgroup(0)]],- uint tgpig[[threadgroup_position_in_grid]],- uint tpitg[[thread_position_in_threadgroup]],- uint ntg[[threads_per_threadgroup]]) {- device const float * x = (device const float *) ((device const char *) src0 + tgpig*nb01);- // MEAN- // parallel sum- sum[tpitg] = 0.0f;- for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- sum[tpitg] += x[i00];- }- // reduce- threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint i = ntg/2; i > 0; i /= 2) {- if (tpitg < i) {- sum[tpitg] += sum[tpitg + i];- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }- const float mean = sum[0] / ne00;-- // recenter and VARIANCE- threadgroup_barrier(mem_flags::mem_threadgroup);- device float * y = dst + tgpig*ne00;- sum[tpitg] = 0.0f;- for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- y[i00] = x[i00] - mean;- sum[tpitg] += y[i00] * y[i00];- }-- // reduce- threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint i = ntg/2; i > 0; i /= 2) {- if (tpitg < i) {- sum[tpitg] += sum[tpitg + i];- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }- const float variance = sum[0] / ne00;-- const float scale = 1.0f/sqrt(variance + eps);- for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- y[i00] = y[i00] * scale;- }-}--kernel void kernel_rms_norm(- device const void * src0,- device float * dst,- constant int64_t & ne00,- constant uint64_t & nb01,- constant float & eps,- threadgroup float * buf [[threadgroup(0)]],- uint tgpig[[threadgroup_position_in_grid]],- uint tpitg[[thread_position_in_threadgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]],- uint tiisg[[thread_index_in_simdgroup]],- uint ntg[[threads_per_threadgroup]]) {- device const float4 * x = (device const float4 *) ((device const char *) src0 + tgpig*nb01);-- float4 sumf = 0;- float all_sum = 0;-- // parallel sum- for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- sumf += x[i00] * x[i00];- }- all_sum = sumf[0] + sumf[1] + sumf[2] + sumf[3];- all_sum = simd_sum(all_sum);- if (ntg > N_SIMDWIDTH) {- if (sgitg == 0) {- buf[tiisg] = 0.0f;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (tiisg == 0) {- buf[sgitg] = all_sum;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- all_sum = buf[tiisg];- all_sum = simd_sum(all_sum);- }-- const float mean = all_sum/ne00;- const float scale = 1.0f/sqrt(mean + eps);-- device float4 * y = (device float4 *) (dst + tgpig*ne00);- for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- y[i00] = x[i00] * scale;- }-}--kernel void kernel_group_norm(- device const float * src0,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int32_t & n_groups,- constant float & eps,- threadgroup float * buf [[threadgroup(0)]],- uint tgpig[[threadgroup_position_in_grid]],- uint tpitg[[thread_position_in_threadgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]],- uint tiisg[[thread_index_in_simdgroup]],- uint ntg[[threads_per_threadgroup]]) {- const int64_t ne = ne00*ne01*ne02;- const int64_t gs = ne00*ne01*((ne02 + n_groups - 1) / n_groups);-- int start = tgpig * gs;- int end = start + gs;-- start += tpitg;-- if (end >= ne) {- end = ne;- }-- float tmp = 0.0f; // partial sum for thread in warp-- for (int j = start; j < end; j += ntg) {- tmp += src0[j];- }-- threadgroup_barrier(mem_flags::mem_threadgroup);- tmp = simd_sum(tmp);- if (ntg > N_SIMDWIDTH) {- if (sgitg == 0) {- buf[tiisg] = 0.0f;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (tiisg == 0) {- buf[sgitg] = tmp;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- tmp = buf[tiisg];- tmp = simd_sum(tmp);- }-- const float mean = tmp / gs;- tmp = 0.0f;-- for (int j = start; j < end; j += ntg) {- float xi = src0[j] - mean;- dst[j] = xi;- tmp += xi * xi;- }-- tmp = simd_sum(tmp);- if (ntg > N_SIMDWIDTH) {- if (sgitg == 0) {- buf[tiisg] = 0.0f;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (tiisg == 0) {- buf[sgitg] = tmp;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- tmp = buf[tiisg];- tmp = simd_sum(tmp);- }-- const float variance = tmp / gs;- const float scale = 1.0f/sqrt(variance + eps);- for (int j = start; j < end; j += ntg) {- dst[j] *= scale;- }-}--// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])-// il indicates where the q4 quants begin (0 or QK4_0/4)-// we assume that the yl's have been multiplied with the appropriate scale factor-// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)-inline float block_q_n_dot_y(device const block_q4_0 * qb_curr, float sumy, thread float * yl, int il) {- float d = qb_curr->d;-- float2 acc = 0.f;-- device const uint16_t * qs = ((device const uint16_t *)qb_curr + 1 + il/2);-- for (int i = 0; i < 8; i+=2) {- acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F)- + yl[i + 1] * (qs[i / 2] & 0x0F00);- acc[1] += yl[i + 8] * (qs[i / 2] & 0x00F0)- + yl[i + 9] * (qs[i / 2] & 0xF000);- }- return d * (sumy * -8.f + acc[0] + acc[1]);-}--// function for calculate inner product between half a q4_1 block and 16 floats (yl), sumy is SUM(yl[i])-// il indicates where the q4 quants begin (0 or QK4_0/4)-// we assume that the yl's have been multiplied with the appropriate scale factor-// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)-inline float block_q_n_dot_y(device const block_q4_1 * qb_curr, float sumy, thread float * yl, int il) {- float d = qb_curr->d;- float m = qb_curr->m;-- float2 acc = 0.f;-- device const uint16_t * qs = ((device const uint16_t *)qb_curr + 2 + il/2);-- for (int i = 0; i < 8; i+=2) {- acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F)- + yl[i + 1] * (qs[i / 2] & 0x0F00);- acc[1] += yl[i + 8] * (qs[i / 2] & 0x00F0)- + yl[i + 9] * (qs[i / 2] & 0xF000);- }- return d * (acc[0] + acc[1]) + sumy * m;-}--// function for calculate inner product between half a q5_0 block and 16 floats (yl), sumy is SUM(yl[i])-// il indicates where the q5 quants begin (0 or QK5_0/4)-// we assume that the yl's have been multiplied with the appropriate scale factor-// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)-inline float block_q_n_dot_y(device const block_q5_0 * qb_curr, float sumy, thread float * yl, int il) {- float d = qb_curr->d;-- float2 acc = 0.f;-- device const uint16_t * qs = ((device const uint16_t *)qb_curr + 3 + il/2);- const uint32_t qh = *((device const uint32_t *)qb_curr->qh);-- for (int i = 0; i < 8; i+=2) {- acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010))- + yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));- acc[1] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100))- + yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));- }- return d * (sumy * -16.f + acc[0] + acc[1]);-}--// function for calculate inner product between half a q5_1 block and 16 floats (yl), sumy is SUM(yl[i])-// il indicates where the q5 quants begin (0 or QK5_1/4)-// we assume that the yl's have been multiplied with the appropriate scale factor-// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)-inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thread float * yl, int il) {- float d = qb_curr->d;- float m = qb_curr->m;-- float2 acc = 0.f;-- device const uint16_t * qs = ((device const uint16_t *)qb_curr + 4 + il/2);- const uint32_t qh = *((device const uint32_t *)qb_curr->qh);-- for (int i = 0; i < 8; i+=2) {- acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010))- + yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));- acc[1] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100))- + yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));- }- return d * (acc[0] + acc[1]) + sumy * m;-}--// putting them in the kernel cause a significant performance penalty-#define N_DST 4 // each SIMD group works on 4 rows-#define N_SIMDGROUP 2 // number of SIMD groups in a thread group-//Note: This is a template, but strictly speaking it only applies to-// quantizations where the block size is 32. It also does not-// guard against the number of rows not being divisible by-// N_DST, so this is another explicit assumption of the implementation.-template<typename block_q_type, int nr, int nsg, int nw>-void mul_vec_q_n_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- int64_t ne00,- int64_t ne01,- int64_t ne02,- int64_t ne10,- int64_t ne12,- int64_t ne0,- int64_t ne1,- uint r2,- uint r3,- uint3 tgpig, uint tiisg, uint sgitg) {- const int nb = ne00/QK4_0;-- const int r0 = tgpig.x;- const int r1 = tgpig.y;- const int im = tgpig.z;-- const int first_row = (r0 * nsg + sgitg) * nr;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q_type * x = (device const block_q_type *) src0 + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float yl[16]; // src1 vector cache- float sumf[nr] = {0.f};-- const int ix = (tiisg/2);- const int il = (tiisg%2)*8;-- device const float * yb = y + ix * QK4_0 + il;-- // each thread in a SIMD group deals with half a block.- for (int ib = ix; ib < nb; ib += nw/2) {- float sumy = 0;- for (int i = 0; i < 8; i += 2) {- sumy += yb[i] + yb[i+1];- yl[i+0] = yb[i+ 0];- yl[i+1] = yb[i+ 1]/256.f;-- sumy += yb[i+16] + yb[i+17];- yl[i+8] = yb[i+16]/16.f;- yl[i+9] = yb[i+17]/4096.f;- }-- for (int row = 0; row < nr; row++) {- sumf[row] += block_q_n_dot_y(x+ib+row*nb, sumy, yl, il);- }-- yb += QK4_0 * 16;- }-- for (int row = 0; row < nr; ++row) {- const float tot = simd_sum(sumf[row]);- if (tiisg == 0 && first_row + row < ne01) {- dst[im*ne0*ne1 + r1*ne0 + first_row + row] = tot;- }- }-}--kernel void kernel_mul_mv_q4_0_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32_impl<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);-}--kernel void kernel_mul_mv_q4_1_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32_impl<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);-}--kernel void kernel_mul_mv_q5_0_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32_impl<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);-}--kernel void kernel_mul_mv_q5_1_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {- mul_vec_q_n_f32_impl<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);-}---#define NB_Q8_0 8--void kernel_mul_mv_q8_0_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {- const int nr = N_DST;- const int nsg = N_SIMDGROUP;- const int nw = N_SIMDWIDTH;-- const int nb = ne00/QK8_0;- const int r0 = tgpig.x;- const int r1 = tgpig.y;- const int im = tgpig.z;-- const int first_row = (r0 * nsg + sgitg) * nr;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q8_0 * x = (device const block_q8_0 *) src0 + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float yl[NB_Q8_0];- float sumf[nr]={0.f};-- const int ix = tiisg/4;- const int il = tiisg%4;-- device const float * yb = y + ix * QK8_0 + NB_Q8_0*il;-- // each thread in a SIMD group deals with NB_Q8_0 quants at a time- for (int ib = ix; ib < nb; ib += nw/4) {- for (int i = 0; i < NB_Q8_0; ++i) {- yl[i] = yb[i];- }-- for (int row = 0; row < nr; row++) {- device const int8_t * qs = x[ib+row*nb].qs + NB_Q8_0*il;- float sumq = 0.f;- for (int iq = 0; iq < NB_Q8_0; ++iq) {- sumq += qs[iq] * yl[iq];- }- sumf[row] += sumq*x[ib+row*nb].d;- }-- yb += NB_Q8_0 * nw;- }-- for (int row = 0; row < nr; ++row) {- const float tot = simd_sum(sumf[row]);- if (tiisg == 0 && first_row + row < ne01) {- dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;- }- }-}--[[host_name("kernel_mul_mv_q8_0_f32")]]-kernel void kernel_mul_mv_q8_0_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {- kernel_mul_mv_q8_0_f32_impl(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);-}--#define N_F32_F32 4--void kernel_mul_mv_f32_f32_impl(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {-- const int64_t r0 = tgpig.x;- const int64_t rb = tgpig.y*N_F32_F32;- const int64_t im = tgpig.z;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;-- device const float * x = (device const float *) (src0 + offset0);-- if (ne00 < 128) {- for (int row = 0; row < N_F32_F32; ++row) {- int r1 = rb + row;- if (r1 >= ne11) {- break;- }-- device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);-- float sumf = 0;- for (int i = tiisg; i < ne00; i += 32) {- sumf += (float) x[i] * (float) y[i];- }-- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }- } else {- device const float4 * x4 = (device const float4 *)x;- for (int row = 0; row < N_F32_F32; ++row) {- int r1 = rb + row;- if (r1 >= ne11) {- break;- }-- device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);- device const float4 * y4 = (device const float4 *) y;-- float sumf = 0;- for (int i = tiisg; i < ne00/4; i += 32) {- for (int k = 0; k < 4; ++k) sumf += (float) x4[i][k] * y4[i][k];- }-- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) x[i] * y[i];- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }- }-}--[[host_name("kernel_mul_mv_f32_f32")]]-kernel void kernel_mul_mv_f32_f32(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {- kernel_mul_mv_f32_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);-}--#define N_F16_F16 4--kernel void kernel_mul_mv_f16_f16(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {-- const int64_t r0 = tgpig.x;- const int64_t rb = tgpig.y*N_F16_F16;- const int64_t im = tgpig.z;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;-- device const half * x = (device const half *) (src0 + offset0);-- if (ne00 < 128) {- for (int row = 0; row < N_F16_F16; ++row) {- int r1 = rb + row;- if (r1 >= ne11) {- break;- }-- device const half * y = (device const half *) (src1 + r1*nb11 + im*nb12);-- float sumf = 0;- for (int i = tiisg; i < ne00; i += 32) {- sumf += (half) x[i] * (half) y[i];- }-- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }- } else {- device const half4 * x4 = (device const half4 *)x;- for (int row = 0; row < N_F16_F16; ++row) {- int r1 = rb + row;- if (r1 >= ne11) {- break;- }-- device const half * y = (device const half *) (src1 + r1*nb11 + im*nb12);- device const half4 * y4 = (device const half4 *) y;-- float sumf = 0;- for (int i = tiisg; i < ne00/4; i += 32) {- for (int k = 0; k < 4; ++k) sumf += (half) x4[i][k] * y4[i][k];- }-- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (half) x[i] * y[i];- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }- }-}--void kernel_mul_mv_f16_f32_1row_impl(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {-- const int64_t r0 = tgpig.x;- const int64_t r1 = tgpig.y;- const int64_t im = tgpig.z;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;-- device const half * x = (device const half *) (src0 + offset0);- device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);-- float sumf = 0;- if (ne00 < 128) {- for (int i = tiisg; i < ne00; i += 32) {- sumf += (float) x[i] * (float) y[i];- }- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- } else {- device const half4 * x4 = (device const half4 *) x;- device const float4 * y4 = (device const float4 *) y;- for (int i = tiisg; i < ne00/4; i += 32) {- for (int k = 0; k < 4; ++k) sumf += (float)x4[i][k] * y4[i][k];- }- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) x[i] * y[i];- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }-}--[[host_name("kernel_mul_mv_f16_f32_1row")]]-kernel void kernel_mul_mv_f16_f32_1row(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {- kernel_mul_mv_f16_f32_1row_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);-}--#define N_F16_F32 4--void kernel_mul_mv_f16_f32_impl(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {-- const int64_t r0 = tgpig.x;- const int64_t rb = tgpig.y*N_F16_F32;- const int64_t im = tgpig.z;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;-- device const half * x = (device const half *) (src0 + offset0);-- if (ne00 < 128) {- for (int row = 0; row < N_F16_F32; ++row) {- int r1 = rb + row;- if (r1 >= ne11) {- break;- }-- device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);-- float sumf = 0;- for (int i = tiisg; i < ne00; i += 32) {- sumf += (float) x[i] * (float) y[i];- }-- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }- } else {- device const half4 * x4 = (device const half4 *)x;- for (int row = 0; row < N_F16_F32; ++row) {- int r1 = rb + row;- if (r1 >= ne11) {- break;- }-- device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);- device const float4 * y4 = (device const float4 *) y;-- float sumf = 0;- for (int i = tiisg; i < ne00/4; i += 32) {- for (int k = 0; k < 4; ++k) sumf += (float) x4[i][k] * y4[i][k];- }-- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) x[i] * y[i];- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }- }-}--[[host_name("kernel_mul_mv_f16_f32")]]-kernel void kernel_mul_mv_f16_f32(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {- kernel_mul_mv_f16_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);-}--// Assumes row size (ne00) is a multiple of 4-kernel void kernel_mul_mv_f16_f32_l4(- device const char * src0,- device const char * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]]) {-- const int nrows = ne11;- const int64_t r0 = tgpig.x;- const int64_t im = tgpig.z;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;-- device const half4 * x4 = (device const half4 *) (src0 + offset0);-- for (int r1 = 0; r1 < nrows; ++r1) {- device const float4 * y4 = (device const float4 *) (src1 + r1*nb11 + im*nb12);-- float sumf = 0;- for (int i = tiisg; i < ne00/4; i += 32) {- for (int k = 0; k < 4; ++k) sumf += (float) x4[i][k] * y4[i][k];- }-- float all_sum = simd_sum(sumf);- if (tiisg == 0) {- dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;- }- }-}--kernel void kernel_alibi_f32(- device const float * src0,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- constant float & m0,- constant float & m1,- constant int & n_heads_log2_floor,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- //const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);-- const int64_t k = i3*ne3 + i2;-- float m_k;- if (k < n_heads_log2_floor) {- m_k = pow(m0, k + 1);- } else {- m_k = pow(m1, 2 * (k - n_heads_log2_floor) + 1);- }-- device char * dst_row = (device char *) dst + i3*nb3 + i2*nb2 + i1*nb1;- device const char * src_row = (device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01;- for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {- const float src_v = *(device float *)(src_row + i00*nb00);- device float * dst_v = (device float *)(dst_row + i00*nb0);- *dst_v = i00 * m_k + src_v;- }-}--static float rope_yarn_ramp(const float low, const float high, const int i0) {- const float y = (i0 / 2 - low) / max(0.001f, high - low);- return 1.0f - min(1.0f, max(0.0f, y));-}--// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn-// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.-static void rope_yarn(- float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,- thread float * cos_theta, thread float * sin_theta-) {- // Get n-d rotational scaling corrected for extrapolation- float theta_interp = freq_scale * theta_extrap;- float theta = theta_interp;- if (ext_factor != 0.0f) {- float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;- theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;-- // Get n-d magnitude scaling corrected for interpolation- mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);- }- *cos_theta = cos(theta) * mscale;- *sin_theta = sin(theta) * mscale;-}--// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get-// `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`-static float rope_yarn_corr_factor(int n_dims, int n_orig_ctx, float n_rot, float base) {- return n_dims * log(n_orig_ctx / (n_rot * 2 * M_PI_F)) / (2 * log(base));-}--static void rope_yarn_corr_dims(- int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]-) {- // start and end correction dims- dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_fast, freq_base)));- dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_slow, freq_base)));-}--typedef void (rope_t)(- device const void * src0,- device const int32_t * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- constant int & n_past,- constant int & n_dims,- constant int & mode,- constant int & n_orig_ctx,- constant float & freq_base,- constant float & freq_scale,- constant float & ext_factor,- constant float & attn_factor,- constant float & beta_fast,- constant float & beta_slow,- uint tiitg[[thread_index_in_threadgroup]],- uint3 tptg[[threads_per_threadgroup]],- uint3 tgpig[[threadgroup_position_in_grid]]);--template<typename T>-kernel void kernel_rope(- device const void * src0,- device const int32_t * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- constant int & n_past,- constant int & n_dims,- constant int & mode,- constant int & n_orig_ctx,- constant float & freq_base,- constant float & freq_scale,- constant float & ext_factor,- constant float & attn_factor,- constant float & beta_fast,- constant float & beta_slow,- uint tiitg[[thread_index_in_threadgroup]],- uint3 tptg[[threads_per_threadgroup]],- uint3 tgpig[[threadgroup_position_in_grid]]) {- const int64_t i3 = tgpig[2];- const int64_t i2 = tgpig[1];- const int64_t i1 = tgpig[0];-- const bool is_neox = mode & 2;-- float corr_dims[2];- rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);-- device const int32_t * pos = src1;-- const int64_t p = pos[i2];-- const float theta_0 = (float)p;- const float inv_ndims = -1.f/n_dims;-- if (!is_neox) {- for (int64_t i0 = 2*tiitg; i0 < ne0; i0 += 2*tptg.x) {-- const float theta = theta_0 * pow(freq_base, inv_ndims*i0);- float cos_theta, sin_theta;- rope_yarn(theta, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);-- device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);- device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- const T x0 = src[0];- const T x1 = src[1];-- dst_data[0] = x0*cos_theta - x1*sin_theta;- dst_data[1] = x0*sin_theta + x1*cos_theta;- }- } else {- for (int64_t ic = 2*tiitg; ic < ne0; ic += 2*tptg.x) {- if (ic < n_dims) {- const int64_t ib = 0;-- // simplified from `(ib * n_dims + ic) * inv_ndims`- const float cur_rot = inv_ndims*ic - ib;-- const float theta = theta_0 * pow(freq_base, cur_rot);- float cos_theta, sin_theta;- rope_yarn(theta, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);-- const int64_t i0 = ib*n_dims + ic/2;-- device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);- device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- const float x0 = src[0];- const float x1 = src[n_dims/2];-- dst_data[0] = x0*cos_theta - x1*sin_theta;- dst_data[n_dims/2] = x0*sin_theta + x1*cos_theta;- } else {- const int64_t i0 = ic;-- device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);- device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- dst_data[0] = src[0];- dst_data[1] = src[1];- }- }- }-}--template [[host_name("kernel_rope_f32")]] kernel rope_t kernel_rope<float>;-template [[host_name("kernel_rope_f16")]] kernel rope_t kernel_rope<half>;--typedef void (im2col_t)(- device const float * x,- device char * dst,- constant int32_t & ofs0,- constant int32_t & ofs1,- constant int32_t & IW,- constant int32_t & IH,- constant int32_t & CHW,- constant int32_t & s0,- constant int32_t & s1,- constant int32_t & p0,- constant int32_t & p1,- constant int32_t & d0,- constant int32_t & d1,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tgpg[[threadgroups_per_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]);--template <typename T>-kernel void kernel_im2col(- device const float * x,- device char * dst,- constant int32_t & ofs0,- constant int32_t & ofs1,- constant int32_t & IW,- constant int32_t & IH,- constant int32_t & CHW,- constant int32_t & s0,- constant int32_t & s1,- constant int32_t & p0,- constant int32_t & p1,- constant int32_t & d0,- constant int32_t & d1,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tgpg[[threadgroups_per_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int32_t iiw = tgpig[2] * s0 + tpitg[2] * d0 - p0;- const int32_t iih = tgpig[1] * s1 + tpitg[1] * d1 - p1;-- const int32_t offset_dst =- (tpitg[0] * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * CHW +- (tgpig[0] * (ntg[1] * ntg[2]) + tpitg[1] * ntg[2] + tpitg[2]);-- device T * pdst = (device T *) (dst);-- if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {- pdst[offset_dst] = 0.0f;- } else {- const int32_t offset_src = tpitg[0] * ofs0 + tgpig[0] * ofs1;- pdst[offset_dst] = x[offset_src + iih * IW + iiw];- }-}--template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col<float>;-template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col<half>;--kernel void kernel_upscale_f32(- device const char * src0,- device char * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- constant int32_t & sf,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {-- const int64_t i3 = tgpig.z;- const int64_t i2 = tgpig.y;- const int64_t i1 = tgpig.x;-- const int64_t i03 = i3;- const int64_t i02 = i2;- const int64_t i01 = i1/sf;-- device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);- device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1);-- for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- dst_ptr[i0] = src0_ptr[i0/sf];- }-}--kernel void kernel_pad_f32(- device const char * src0,- device char * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {-- const int64_t i3 = tgpig.z;- const int64_t i2 = tgpig.y;- const int64_t i1 = tgpig.x;-- const int64_t i03 = i3;- const int64_t i02 = i2;- const int64_t i01 = i1;-- device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);- device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1);-- if (i1 < ne01 && i2 < ne02 && i3 < ne03) {- for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- if (i0 < ne00) {- dst_ptr[i0] = src0_ptr[i0];- } else {- dst_ptr[i0] = 0.0f;- }- }-- return;- }-- for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- dst_ptr[i0] = 0.0f;- }-}--// bitonic sort implementation following the CUDA kernels as reference-typedef void (argsort_t)(- device const float * x,- device int32_t * dst,- constant int64_t & ncols,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]]);--template<ggml_sort_order order>-kernel void kernel_argsort_f32_i32(- device const float * x,- device int32_t * dst,- constant int64_t & ncols,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]]) {- // bitonic sort- int col = tpitg[0];- int row = tgpig[1];-- if (col >= ncols) return;-- device const float * x_row = x + row * ncols;- device int32_t * dst_row = dst + row * ncols;-- // initialize indices- if (col < ncols) {- dst_row[col] = col;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- for (int k = 2; k <= ncols; k *= 2) {- for (int j = k / 2; j > 0; j /= 2) {- int ixj = col ^ j;- if (ixj > col) {- if ((col & k) == 0) {- if (order == GGML_SORT_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) {- SWAP(dst_row[col], dst_row[ixj]);- }- } else {- if (order == GGML_SORT_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) {- SWAP(dst_row[col], dst_row[ixj]);- }- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }- }-}--template [[host_name("kernel_argsort_f32_i32_asc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_ASC>;-template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_DESC>;--kernel void kernel_leaky_relu_f32(- device const float * src0,- device float * dst,- constant float & slope,- uint tpig[[thread_position_in_grid]]) {- dst[tpig] = src0[tpig] > 0.0f ? src0[tpig] : src0[tpig] * slope;-}--kernel void kernel_cpy_f16_f16(- device const half * src0,- device half * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);-- device half * dst_data = (device half *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {- device const half * src = (device half *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);- dst_data[i00] = src[0];- }-}--kernel void kernel_cpy_f16_f32(- device const half * src0,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);-- device float * dst_data = (device float *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {- device const half * src = (device half *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);- dst_data[i00] = src[0];- }-}--kernel void kernel_cpy_f32_f16(- device const float * src0,- device half * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);-- device half * dst_data = (device half *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {- device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);-- dst_data[i00] = src[0];- }-}--kernel void kernel_cpy_f32_f32(- device const float * src0,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);-- device float * dst_data = (device float *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {- device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);-- dst_data[i00] = src[0];- }-}--kernel void kernel_cpy_f32_q8_0(- device const float * src0,- device void * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK8_0;-- device block_q8_0 * dst_data = (device block_q8_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- for (int64_t i00 = tpitg.x*QK8_0; i00 < ne00; i00 += ntg.x*QK8_0) {- device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);-- float amax = 0.0f; // absolute max-- for (int j = 0; j < QK8_0; j++) {- const float v = src[j];- amax = MAX(amax, fabs(v));- }-- const float d = amax / ((1 << 7) - 1);- const float id = d ? 1.0f/d : 0.0f;-- dst_data[i00/QK8_0].d = d;-- for (int j = 0; j < QK8_0; ++j) {- const float x0 = src[j]*id;-- dst_data[i00/QK8_0].qs[j] = round(x0);- }- }-}--kernel void kernel_cpy_f32_q4_0(- device const float * src0,- device void * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_0;-- device block_q4_0 * dst_data = (device block_q4_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- for (int64_t i00 = tpitg.x*QK4_0; i00 < ne00; i00 += ntg.x*QK4_0) {- device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);-- float amax = 0.0f; // absolute max- float max = 0.0f;-- for (int j = 0; j < QK4_0; j++) {- const float v = src[j];- if (amax < fabs(v)) {- amax = fabs(v);- max = v;- }- }-- const float d = max / -8;- const float id = d ? 1.0f/d : 0.0f;-- dst_data[i00/QK4_0].d = d;-- for (int j = 0; j < QK4_0/2; ++j) {- const float x0 = src[0 + j]*id;- const float x1 = src[QK4_0/2 + j]*id;-- const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));- const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));-- dst_data[i00/QK4_0].qs[j] = xi0;- dst_data[i00/QK4_0].qs[j] |= xi1 << 4;- }- }-}--kernel void kernel_cpy_f32_q4_1(- device const float * src0,- device void * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_1;-- device block_q4_1 * dst_data = (device block_q4_1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);-- for (int64_t i00 = tpitg.x*QK4_1; i00 < ne00; i00 += ntg.x*QK4_1) {- device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);-- float min = FLT_MAX;- float max = -FLT_MAX;-- for (int j = 0; j < QK4_1; j++) {- const float v = src[j];- if (min > v) min = v;- if (max < v) max = v;- }-- const float d = (max - min) / ((1 << 4) - 1);- const float id = d ? 1.0f/d : 0.0f;-- dst_data[i00/QK4_1].d = d;- dst_data[i00/QK4_1].m = min;-- for (int j = 0; j < QK4_1/2; ++j) {- const float x0 = (src[0 + j] - min)*id;- const float x1 = (src[QK4_1/2 + j] - min)*id;-- const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));- const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));-- dst_data[i00/QK4_1].qs[j] = xi0;- dst_data[i00/QK4_1].qs[j] |= xi1 << 4;- }- }-}--kernel void kernel_concat(- device const char * src0,- device const char * src1,- device char * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant int64_t & ne13,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant uint64_t & nb13,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {-- const int64_t i03 = tgpig.z;- const int64_t i02 = tgpig.y;- const int64_t i01 = tgpig.x;-- const int64_t i13 = i03 % ne13;- const int64_t i12 = i02 % ne12;- const int64_t i11 = i01 % ne11;-- device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + tpitg.x*nb00;- device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11 + tpitg.x*nb10;- device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + tpitg.x*nb0;-- for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- if (i02 < ne02) {- ((device float *)dst_ptr)[0] = ((device float *)src0_ptr)[0];- src0_ptr += ntg.x*nb00;- } else {- ((device float *)dst_ptr)[0] = ((device float *)src1_ptr)[0];- src1_ptr += ntg.x*nb10;- }- dst_ptr += ntg.x*nb0;- }-}--//============================================ k-quants ======================================================--#ifndef QK_K-#define QK_K 256-#else-static_assert(QK_K == 256 || QK_K == 64, "QK_K must be 256 or 64");-#endif--#if QK_K == 256-#define K_SCALE_SIZE 12-#else-#define K_SCALE_SIZE 4-#endif--typedef struct {- uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits- uint8_t qs[QK_K/4]; // quants- half d; // super-block scale for quantized scales- half dmin; // super-block scale for quantized mins-} block_q2_K;-// 84 bytes / block--typedef struct {- uint8_t hmask[QK_K/8]; // quants - high bit- uint8_t qs[QK_K/4]; // quants - low 2 bits-#if QK_K == 64- uint8_t scales[2];-#else- uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits-#endif- half d; // super-block scale-} block_q3_K;--#if QK_K == 64-typedef struct {- half d[2]; // super-block scales/mins- uint8_t scales[2];- uint8_t qs[QK_K/2]; // 4-bit quants-} block_q4_K;-#else-typedef struct {- half d; // super-block scale for quantized scales- half dmin; // super-block scale for quantized mins- uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits- uint8_t qs[QK_K/2]; // 4--bit quants-} block_q4_K;-#endif--#if QK_K == 64-typedef struct {- half d; // super-block scales/mins- int8_t scales[QK_K/16]; // 8-bit block scales- uint8_t qh[QK_K/8]; // quants, high bit- uint8_t qs[QK_K/2]; // quants, low 4 bits-} block_q5_K;-#else-typedef struct {- half d; // super-block scale for quantized scales- half dmin; // super-block scale for quantized mins- uint8_t scales[3*QK_K/64]; // scales and mins, quantized with 6 bits- uint8_t qh[QK_K/8]; // quants, high bit- uint8_t qs[QK_K/2]; // quants, low 4 bits-} block_q5_K;-// 176 bytes / block-#endif--typedef struct {- uint8_t ql[QK_K/2]; // quants, lower 4 bits- uint8_t qh[QK_K/4]; // quants, upper 2 bits- int8_t scales[QK_K/16]; // scales, quantized with 8 bits- half d; // super-block scale-} block_q6_K;-// 210 bytes / block--typedef struct {- half d;- uint16_t qs[QK_K/8];-} block_iq2_xxs;-// 66 bytes / block for QK_K = 256, so 2.0625 bpw--typedef struct {- half d;- uint16_t qs[QK_K/8];- uint8_t scales[QK_K/32];-} block_iq2_xs;-// 74 bytes / block for QK_K = 256, so 2.3125 bpw--// 2.5625 bpw quants-typedef struct {- half d;- uint8_t qs[QK_K/4];- uint8_t qh[QK_K/32];- uint8_t scales[QK_K/32];-} block_iq2_s;--typedef struct {- half d;- uint8_t qs[3*QK_K/8];-} block_iq3_xxs;-// 98 bytes / block for QK_K = 256, so 3.0625 bpw--// 3.4375 bpw-#if QK_K == 64-#define IQ3S_N_SCALE 2-#else-#define IQ3S_N_SCALE QK_K/64-#endif-typedef struct {- half d;- uint8_t qs[QK_K/4];- uint8_t qh[QK_K/32];- uint8_t signs[QK_K/8];- uint8_t scales[IQ3S_N_SCALE];-} block_iq3_s;--typedef struct {- half d;- uint8_t qs[QK_K/8];- uint8_t scales[QK_K/16];-} block_iq1_s;--// Non-linear quants-#define QK4_NL 32-typedef struct {- half d;- uint8_t qs[QK4_NL/2];-} block_iq4_nl;--typedef struct {- half d;- uint16_t scales_h;- uint8_t scales_l[QK_K/64];- uint8_t qs[QK_K/2];-} block_iq4_xs;--//====================================== dot products =========================--void kernel_mul_mv_q2_K_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- const int nb = ne00/QK_K;- const int r0 = tgpig.x;- const int r1 = tgpig.y;- const int im = tgpig.z;-- const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;- const int ib_row = first_row * nb;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q2_K * x = (device const block_q2_K *) src0 + ib_row + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float yl[32];- float sumf[N_DST]={0.f}, all_sum;-- const int step = sizeof(block_q2_K) * nb;--#if QK_K == 256- const int ix = tiisg/8; // 0...3- const int it = tiisg%8; // 0...7- const int iq = it/4; // 0 or 1- const int ir = it%4; // 0...3- const int is = (8*ir)/16;// 0 or 1-- device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;-- for (int ib = ix; ib < nb; ib += 4) {-- float4 sumy = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; ++i) {- yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];- yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8];- yl[i+16] = y4[i+64]; sumy[2] += yl[i+16];- yl[i+24] = y4[i+96]; sumy[3] += yl[i+24];- }-- device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is;- device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;- device const half * dh = &x[ib].d;-- for (int row = 0; row < N_DST; row++) {-- float4 acc1 = {0.f, 0.f, 0.f, 0.f};- float4 acc2 = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; i += 2) {- acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);- acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);- acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);- acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);- acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);- acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);- acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);- acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);- }- float dall = dh[0];- float dmin = dh[1] * 1.f/16.f;- sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f +- (acc1[1] + 1.f/256.f * acc2[1]) * (sc[2] & 0xF) * 1.f/ 4.f +- (acc1[2] + 1.f/256.f * acc2[2]) * (sc[4] & 0xF) * 1.f/16.f +- (acc1[3] + 1.f/256.f * acc2[3]) * (sc[6] & 0xF) * 1.f/64.f) -- dmin * (sumy[0] * (sc[0] & 0xF0) + sumy[1] * (sc[2] & 0xF0) + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0));-- qs += step/2;- sc += step;- dh += step/2;- }-- y4 += 4 * QK_K;- }-#else- const int ix = tiisg/2; // 0...15- const int it = tiisg%2; // 0...1-- device const float * y4 = y + ix * QK_K + 8 * it;-- for (int ib = ix; ib < nb; ib += 16) {-- float4 sumy = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; ++i) {- yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];- yl[i+ 8] = y4[i+16]; sumy[1] += yl[i+ 8];- yl[i+16] = y4[i+32]; sumy[2] += yl[i+16];- yl[i+24] = y4[i+48]; sumy[3] += yl[i+24];- }-- device const uint8_t * sc = (device const uint8_t *)x[ib].scales;- device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 4 * it;- device const half * dh = &x[ib].d;-- for (int row = 0; row < N_DST; row++) {-- float4 acc1 = {0.f, 0.f, 0.f, 0.f};- float4 acc2 = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; i += 2) {- acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);- acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);- acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);- acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);- acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);- acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);- acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);- acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);- }-- float dall = dh[0];- float dmin = dh[1];- sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f +- (acc1[1] + 1.f/256.f * acc2[1]) * (sc[1] & 0xF) * 1.f/ 4.f +- (acc1[2] + 1.f/256.f * acc2[2]) * (sc[2] & 0xF) * 1.f/16.f +- (acc1[3] + 1.f/256.f * acc2[3]) * (sc[3] & 0xF) * 1.f/64.f) -- dmin * (sumy[0] * (sc[0] >> 4) + sumy[1] * (sc[1] >> 4) + sumy[2] * (sc[2] >> 4) + sumy[3] * (sc[3] >> 4));-- qs += step/2;- sc += step;- dh += step/2;- }-- y4 += 16 * QK_K;- }-#endif-- for (int row = 0; row < N_DST; ++row) {- all_sum = simd_sum(sumf[row]);- if (tiisg == 0) {- dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;- }- }-}--[[host_name("kernel_mul_mv_q2_K_f32")]]-kernel void kernel_mul_mv_q2_K_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- kernel_mul_mv_q2_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);-}--#if QK_K == 256-void kernel_mul_mv_q3_K_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- const int nb = ne00/QK_K;-- const int64_t r0 = tgpig.x;- const int64_t r1 = tgpig.y;- const int64_t im = tgpig.z;-- const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q3_K * x = (device const block_q3_K *) src0 + first_row*nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float yl[32];-- //const uint16_t kmask1 = 0x3030;- //const uint16_t kmask2 = 0x0f0f;-- const int tid = tiisg/4;- const int ix = tiisg%4;- const int ip = tid/4; // 0 or 1- const int il = 2*((tid%4)/2); // 0 or 2- const int ir = tid%2;- const int n = 8;- const int l0 = n*ir;-- // One would think that the Metal compiler would figure out that ip and il can only have- // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it- // with these two tales.- //- // Possible masks for the high bit- const ushort4 mm[4] = {{0x0001, 0x0100, 0x0002, 0x0200}, // ip = 0, il = 0- {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2- {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0- {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2-- // Possible masks for the low 2 bits- const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}};-- const ushort4 hm = mm[2*ip + il/2];-- const int shift = 2*il;- const float v1 = il == 0 ? 4.f : 64.f;- const float v2 = 4.f * v1;-- const uint16_t s_shift1 = 4*ip;- const uint16_t s_shift2 = s_shift1 + il;-- const int q_offset = 32*ip + l0;- const int y_offset = 128*ip + 32*il + l0;-- const int step = sizeof(block_q3_K) * nb / 2;-- device const float * y1 = yy + ix*QK_K + y_offset;-- uint32_t scales32, aux32;- thread uint16_t * scales16 = (thread uint16_t *)&scales32;- thread const int8_t * scales = (thread const int8_t *)&scales32;-- float sumf1[2] = {0.f};- float sumf2[2] = {0.f};- for (int i = ix; i < nb; i += 4) {-- for (int l = 0; l < 8; ++l) {- yl[l+ 0] = y1[l+ 0];- yl[l+ 8] = y1[l+16];- yl[l+16] = y1[l+32];- yl[l+24] = y1[l+48];- }-- device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset);- device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0);- device const uint16_t * a = (device const uint16_t *)(x[i].scales);- device const half * dh = &x[i].d;-- for (int row = 0; row < 2; ++row) {-- const float d_all = (float)dh[0];-- scales16[0] = a[4];- scales16[1] = a[5];- aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030;- scales16[0] = a[il+0];- scales16[1] = a[il+1];- scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32;-- float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;- for (int l = 0; l < n; l += 2) {- const int32_t qs = q[l/2];- s1 += yl[l+0] * (qs & qm[il/2][0]);- s2 += yl[l+1] * (qs & qm[il/2][1]);- s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]);- s4 += yl[l+16] * (qs & qm[il/2][2]);- s5 += yl[l+17] * (qs & qm[il/2][3]);- s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]);- }- float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);- float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);- sumf1[row] += d1 * (scales[0] - 32);- sumf2[row] += d2 * (scales[2] - 32);-- s1 = s2 = s3 = s4 = s5 = s6 = 0;- for (int l = 0; l < n; l += 2) {- const int32_t qs = q[l/2+8];- s1 += yl[l+8] * (qs & qm[il/2][0]);- s2 += yl[l+9] * (qs & qm[il/2][1]);- s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]);- s4 += yl[l+24] * (qs & qm[il/2][2]);- s5 += yl[l+25] * (qs & qm[il/2][3]);- s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]);- }- d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);- d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);- sumf1[row] += d1 * (scales[1] - 32);- sumf2[row] += d2 * (scales[3] - 32);-- q += step;- h += step;- a += step;- dh += step;-- }-- y1 += 4 * QK_K;-- }-- for (int row = 0; row < 2; ++row) {- const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift);- sumf1[row] = simd_sum(sumf);- }- if (tiisg == 0) {- for (int row = 0; row < 2; ++row) {- dst[r1*ne0 + im*ne0*ne1 + first_row + row] = sumf1[row];- }- }-}-#else-void kernel_mul_mv_q3_K_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- const int nb = ne00/QK_K;-- const int64_t r0 = tgpig.x;- const int64_t r1 = tgpig.y;- const int64_t im = tgpig.z;-- const int row = 2 * r0 + sgitg;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q3_K * x = (device const block_q3_K *) src0 + row*nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- const int ix = tiisg/4;- const int il = 4 * (tiisg%4);// 0, 4, 8, 12- const int iq = il/8; // 0, 0, 1, 1- const int in = il%8; // 0, 4, 0, 4-- float2 sum = {0.f, 0.f};-- for (int i = ix; i < nb; i += 8) {-- const float d_all = (float)(x[i].d);-- device const uint16_t * q = (device const uint16_t *)(x[i].qs + il);- device const uint16_t * h = (device const uint16_t *)(x[i].hmask + in);- device const uint16_t * s = (device const uint16_t *)(x[i].scales);- device const float * y = yy + i * QK_K + il;-- const float d1 = d_all * ((int32_t)(s[0] & 0x000F) - 8);- const float d2 = d_all * ((int32_t)(s[0] & 0x00F0) - 128) * 1.f/64.f;- const float d3 = d_all * ((int32_t)(s[0] & 0x0F00) - 2048) * 1.f/4096.f;- const float d4 = d_all * ((int32_t)(s[0] & 0xF000) - 32768) * 1.f/262144.f;-- for (int l = 0; l < 4; l += 2) {- const uint16_t hm = h[l/2] >> iq;- sum[0] += y[l+ 0] * d1 * ((int32_t)(q[l/2] & 0x0003) - ((hm & 0x0001) ? 0 : 4))- + y[l+16] * d2 * ((int32_t)(q[l/2] & 0x000c) - ((hm & 0x0004) ? 0 : 16))- + y[l+32] * d3 * ((int32_t)(q[l/2] & 0x0030) - ((hm & 0x0010) ? 0 : 64))- + y[l+48] * d4 * ((int32_t)(q[l/2] & 0x00c0) - ((hm & 0x0040) ? 0 : 256));- sum[1] += y[l+ 1] * d1 * ((int32_t)(q[l/2] & 0x0300) - ((hm & 0x0100) ? 0 : 1024))- + y[l+17] * d2 * ((int32_t)(q[l/2] & 0x0c00) - ((hm & 0x0400) ? 0 : 4096))- + y[l+33] * d3 * ((int32_t)(q[l/2] & 0x3000) - ((hm & 0x1000) ? 0 : 16384))- + y[l+49] * d4 * ((int32_t)(q[l/2] & 0xc000) - ((hm & 0x4000) ? 0 : 65536));- }-- }- const float sumf = sum[0] + sum[1] * 1.f/256.f;-- const float tot = simd_sum(sumf);- if (tiisg == 0) {- dst[r1*ne0 + im*ne0*ne1 + row] = tot;- }--}-#endif--[[host_name("kernel_mul_mv_q3_K_f32")]]-kernel void kernel_mul_mv_q3_K_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- kernel_mul_mv_q3_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);-}--#if QK_K == 256-void kernel_mul_mv_q4_K_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;-- const int ix = tiisg/8; // 0...3- const int it = tiisg%8; // 0...7- const int iq = it/4; // 0 or 1- const int ir = it%4; // 0...3-- const int nb = ne00/QK_K;- const int r0 = tgpig.x;- const int r1 = tgpig.y;- const int im = tgpig.z;- //const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;- const int first_row = r0 * N_DST;- const int ib_row = first_row * nb;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float yl[16];- float yh[16];- float sumf[N_DST]={0.f}, all_sum;-- const int step = sizeof(block_q4_K) * nb / 2;-- device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;-- uint16_t sc16[4];- thread const uint8_t * sc8 = (thread const uint8_t *)sc16;-- for (int ib = ix; ib < nb; ib += 4) {-- float4 sumy = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; ++i) {- yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0];- yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8];- yh[i+0] = y4[i+128]; sumy[2] += yh[i+0];- yh[i+8] = y4[i+160]; sumy[3] += yh[i+8];- }-- device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq;- device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;- device const half * dh = &x[ib].d;-- for (int row = 0; row < N_DST; row++) {-- sc16[0] = sc[0] & kmask1;- sc16[1] = sc[2] & kmask1;- sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);- sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);-- device const uint16_t * q2 = q1 + 32;-- float4 acc1 = {0.f, 0.f, 0.f, 0.f};- float4 acc2 = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; i += 2) {- acc1[0] += yl[i+0] * (q1[i/2] & 0x000F);- acc1[1] += yl[i+1] * (q1[i/2] & 0x0F00);- acc1[2] += yl[i+8] * (q1[i/2] & 0x00F0);- acc1[3] += yl[i+9] * (q1[i/2] & 0xF000);- acc2[0] += yh[i+0] * (q2[i/2] & 0x000F);- acc2[1] += yh[i+1] * (q2[i/2] & 0x0F00);- acc2[2] += yh[i+8] * (q2[i/2] & 0x00F0);- acc2[3] += yh[i+9] * (q2[i/2] & 0xF000);- }-- float dall = dh[0];- float dmin = dh[1];- sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] +- (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f +- (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] +- (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) -- dmin * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);-- q1 += step;- sc += step;- dh += step;- }-- y4 += 4 * QK_K;- }-- for (int row = 0; row < N_DST; ++row) {- all_sum = simd_sum(sumf[row]);- if (tiisg == 0) {- dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;- }- }-}-#else-void kernel_mul_mv_q4_K_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- const int ix = tiisg/4; // 0...7- const int it = tiisg%4; // 0...3-- const int nb = ne00/QK_K;- const int r0 = tgpig.x;- const int r1 = tgpig.y;- const int im = tgpig.z;- const int first_row = r0 * N_DST;- const int ib_row = first_row * nb;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0;- device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float yl[8];- float yh[8];- float sumf[N_DST]={0.f}, all_sum;-- const int step = sizeof(block_q4_K) * nb / 2;-- device const float * y4 = y + ix * QK_K + 8 * it;-- uint16_t sc16[4];-- for (int ib = ix; ib < nb; ib += 8) {-- float2 sumy = {0.f, 0.f};- for (int i = 0; i < 8; ++i) {- yl[i] = y4[i+ 0]; sumy[0] += yl[i];- yh[i] = y4[i+32]; sumy[1] += yh[i];- }-- device const uint16_t * sc = (device const uint16_t *)x[ib].scales;- device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 4 * it;- device const half * dh = x[ib].d;-- for (int row = 0; row < N_DST; row++) {-- sc16[0] = sc[0] & 0x000f;- sc16[1] = sc[0] & 0x0f00;- sc16[2] = sc[0] & 0x00f0;- sc16[3] = sc[0] & 0xf000;-- float2 acc1 = {0.f, 0.f};- float2 acc2 = {0.f, 0.f};- for (int i = 0; i < 8; i += 2) {- acc1[0] += yl[i+0] * (qs[i/2] & 0x000F);- acc1[1] += yl[i+1] * (qs[i/2] & 0x0F00);- acc2[0] += yh[i+0] * (qs[i/2] & 0x00F0);- acc2[1] += yh[i+1] * (qs[i/2] & 0xF000);- }-- float dall = dh[0];- float dmin = dh[1];- sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc1[1]) * sc16[0] +- (acc2[0] + 1.f/256.f * acc2[1]) * sc16[1] * 1.f/4096.f) -- dmin * 1.f/16.f * (sumy[0] * sc16[2] + sumy[1] * sc16[3] * 1.f/256.f);-- qs += step;- sc += step;- dh += step;- }-- y4 += 8 * QK_K;- }-- for (int row = 0; row < N_DST; ++row) {- all_sum = simd_sum(sumf[row]);- if (tiisg == 0) {- dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;- }- }-}-#endif--[[host_name("kernel_mul_mv_q4_K_f32")]]-kernel void kernel_mul_mv_q4_K_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- kernel_mul_mv_q4_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);-}--void kernel_mul_mv_q5_K_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- const int nb = ne00/QK_K;-- const int64_t r0 = tgpig.x;- const int64_t r1 = tgpig.y;- const int im = tgpig.z;-- const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q5_K * x = (device const block_q5_K *) src0 + first_row*nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float sumf[2]={0.f};-- const int step = sizeof(block_q5_K) * nb;--#if QK_K == 256-#- float yl[16], yh[16];-- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;-- const int tid = tiisg/4;- const int ix = tiisg%4;- const int iq = tid/4;- const int ir = tid%4;- const int n = 8;-- const int l0 = n*ir;- const int q_offset = 32*iq + l0;- const int y_offset = 64*iq + l0;-- const uint8_t hm1 = 1u << (2*iq);- const uint8_t hm2 = hm1 << 1;- const uint8_t hm3 = hm1 << 4;- const uint8_t hm4 = hm2 << 4;-- uint16_t sc16[4];- thread const uint8_t * sc8 = (thread const uint8_t *)sc16;-- device const float * y1 = yy + ix*QK_K + y_offset;-- for (int i = ix; i < nb; i += 4) {-- device const uint8_t * q1 = x[i].qs + q_offset;- device const uint8_t * qh = x[i].qh + l0;- device const half * dh = &x[i].d;- device const uint16_t * a = (device const uint16_t *)x[i].scales + iq;-- device const float * y2 = y1 + 128;- float4 sumy = {0.f, 0.f, 0.f, 0.f};- for (int l = 0; l < 8; ++l) {- yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0];- yl[l+8] = y1[l+32]; sumy[1] += yl[l+8];- yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0];- yh[l+8] = y2[l+32]; sumy[3] += yh[l+8];- }-- for (int row = 0; row < 2; ++row) {-- device const uint8_t * q2 = q1 + 64;-- sc16[0] = a[0] & kmask1;- sc16[1] = a[2] & kmask1;- sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2);- sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2);-- float4 acc1 = {0.f};- float4 acc2 = {0.f};- for (int l = 0; l < n; ++l) {- uint8_t h = qh[l];- acc1[0] += yl[l+0] * (q1[l] & 0x0F);- acc1[1] += yl[l+8] * (q1[l] & 0xF0);- acc1[2] += yh[l+0] * (q2[l] & 0x0F);- acc1[3] += yh[l+8] * (q2[l] & 0xF0);- acc2[0] += h & hm1 ? yl[l+0] : 0.f;- acc2[1] += h & hm2 ? yl[l+8] : 0.f;- acc2[2] += h & hm3 ? yh[l+0] : 0.f;- acc2[3] += h & hm4 ? yh[l+8] : 0.f;- }- const float dall = dh[0];- const float dmin = dh[1];- sumf[row] += dall * (sc8[0] * (acc1[0] + 16.f*acc2[0]) +- sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) +- sc8[4] * (acc1[2] + 16.f*acc2[2]) +- sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) -- dmin * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);-- q1 += step;- qh += step;- dh += step/2;- a += step/2;-- }-- y1 += 4 * QK_K;-- }-#else- float yl[8], yh[8];-- const int il = 4 * (tiisg/8); // 0, 4, 8, 12- const int ix = tiisg%8;- const int iq = il/8; // 0, 0, 1, 1- const int in = il%8; // 0, 4, 0, 4-- device const float * y = yy + ix*QK_K + il;-- for (int i = ix; i < nb; i += 8) {-- for (int l = 0; l < 4; ++l) {- yl[l+0] = y[l+ 0];- yl[l+4] = y[l+16];- yh[l+0] = y[l+32];- yh[l+4] = y[l+48];- }-- device const half * dh = &x[i].d;- device const uint8_t * q = x[i].qs + il;- device const uint8_t * h = x[i].qh + in;- device const int8_t * s = x[i].scales;-- for (int row = 0; row < 2; ++row) {-- const float d = dh[0];-- float2 acc = {0.f, 0.f};- for (int l = 0; l < 4; ++l) {- const uint8_t hl = h[l] >> iq;- acc[0] += yl[l+0] * s[0] * ((int16_t)(q[l+ 0] & 0x0F) - (hl & 0x01 ? 0 : 16))- + yl[l+4] * s[1] * ((int16_t)(q[l+16] & 0x0F) - (hl & 0x04 ? 0 : 16));- acc[1] += yh[l+0] * s[2] * ((int16_t)(q[l+ 0] & 0xF0) - (hl & 0x10 ? 0 : 256))- + yh[l+4] * s[3] * ((int16_t)(q[l+16] & 0xF0) - (hl & 0x40 ? 0 : 256));- }- sumf[row] += d * (acc[0] + 1.f/16.f * acc[1]);-- q += step;- h += step;- s += step;- dh += step/2;-- }-- y += 8 * QK_K;- }-#endif-- for (int row = 0; row < 2; ++row) {- const float tot = simd_sum(sumf[row]);- if (tiisg == 0) {- dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;- }- }-}--[[host_name("kernel_mul_mv_q5_K_f32")]]-kernel void kernel_mul_mv_q5_K_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- kernel_mul_mv_q5_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);-}--void kernel_mul_mv_q6_K_f32_impl(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne10,- constant int64_t & ne12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- const uint8_t kmask1 = 0x03;- const uint8_t kmask2 = 0x0C;- const uint8_t kmask3 = 0x30;- const uint8_t kmask4 = 0xC0;-- const int nb = ne00/QK_K;-- const int64_t r0 = tgpig.x;- const int64_t r1 = tgpig.y;- const int im = tgpig.z;-- const int row = 2 * r0 + sgitg;-- const uint i12 = im%ne12;- const uint i13 = im/ne12;-- const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);-- device const block_q6_K * x = (device const block_q6_K *) src0 + row * nb + offset0;- device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;-- float sumf = 0;--#if QK_K == 256- const int tid = tiisg/2;- const int ix = tiisg%2;- const int ip = tid/8; // 0 or 1- const int il = tid%8;- const int n = 4;- const int l0 = n*il;- const int is = 8*ip + l0/16;-- const int y_offset = 128*ip + l0;- const int q_offset_l = 64*ip + l0;- const int q_offset_h = 32*ip + l0;-- for (int i = ix; i < nb; i += 2) {-- device const uint8_t * q1 = x[i].ql + q_offset_l;- device const uint8_t * q2 = q1 + 32;- device const uint8_t * qh = x[i].qh + q_offset_h;- device const int8_t * sc = x[i].scales + is;-- device const float * y = yy + i * QK_K + y_offset;-- const float dall = x[i].d;-- float4 sums = {0.f, 0.f, 0.f, 0.f};- for (int l = 0; l < n; ++l) {- sums[0] += y[l+ 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32);- sums[1] += y[l+32] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32);- sums[2] += y[l+64] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32);- sums[3] += y[l+96] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32);- }-- sumf += dall * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]);-- }--#else- const int ix = tiisg/4;- const int il = 4*(tiisg%4);-- for (int i = ix; i < nb; i += 8) {- device const float * y = yy + i * QK_K + il;- device const uint8_t * ql = x[i].ql + il;- device const uint8_t * qh = x[i].qh + il;- device const int8_t * s = x[i].scales;-- const float d = x[i].d;-- float4 sums = {0.f, 0.f, 0.f, 0.f};- for (int l = 0; l < 4; ++l) {- sums[0] += y[l+ 0] * ((int8_t)((ql[l+ 0] & 0xF) | ((qh[l] & kmask1) << 4)) - 32);- sums[1] += y[l+16] * ((int8_t)((ql[l+16] & 0xF) | ((qh[l] & kmask2) << 2)) - 32);- sums[2] += y[l+32] * ((int8_t)((ql[l+ 0] >> 4) | ((qh[l] & kmask3) >> 0)) - 32);- sums[3] += y[l+48] * ((int8_t)((ql[l+16] >> 4) | ((qh[l] & kmask4) >> 2)) - 32);- }- sumf += d * (sums[0] * s[0] + sums[1] * s[1] + sums[2] * s[2] + sums[3] * s[3]);- }--#endif-- const float tot = simd_sum(sumf);- if (tiisg == 0) {- dst[r1*ne0 + im*ne0*ne1 + row] = tot;- }-}--[[host_name("kernel_mul_mv_q6_K_f32")]]-kernel void kernel_mul_mv_q6_K_f32(- device const void * src0,- device const float * src1,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant int64_t & ne10,- constant int64_t & ne11,- constant int64_t & ne12,- constant uint64_t & nb10,- constant uint64_t & nb11,- constant uint64_t & nb12,- constant int64_t & ne0,- constant int64_t & ne1,- constant uint & r2,- constant uint & r3,- uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {-- kernel_mul_mv_q6_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);-}--// ======================= "True" 2-bit--constexpr constant static uint64_t iq2xxs_grid[256] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,- 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,- 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,- 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,- 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,- 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,- 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,- 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,- 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,- 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,- 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,- 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,- 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,- 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,- 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,- 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,- 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,- 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,- 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,- 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,- 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,- 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,- 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,- 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,- 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,- 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,- 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,- 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,- 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,- 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,- 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,- 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,- 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,- 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,- 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,- 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,- 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,- 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,- 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,- 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,- 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,- 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,- 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,- 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,- 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,- 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,- 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,- 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,- 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,- 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,- 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,- 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,- 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,- 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,- 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,- 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,- 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,- 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,- 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,- 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,- 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,- 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,- 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,-};--constexpr constant static uint64_t iq2xs_grid[512] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,- 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,- 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,- 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,- 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,- 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,- 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,- 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,- 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,- 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,- 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,- 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,- 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,- 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,- 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,- 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,- 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,- 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,- 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,- 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,- 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,- 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,- 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,- 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,- 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,- 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,- 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,- 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,- 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,- 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,- 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,- 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,- 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,- 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,- 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,- 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,- 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,- 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,- 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,- 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,- 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,- 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,- 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,- 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,- 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,- 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,- 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,- 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,- 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,- 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,- 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,- 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,- 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,- 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,- 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,- 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,- 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,- 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,- 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,- 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,- 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,- 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,- 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,- 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,- 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,- 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,- 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,- 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,- 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,- 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,- 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,- 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,- 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,- 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,- 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,- 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,- 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,- 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,- 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,- 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,- 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,- 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,- 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,- 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,- 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,- 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,- 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,- 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,- 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,- 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,- 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,- 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,- 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,- 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,- 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,- 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,- 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,- 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,- 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,- 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,- 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,- 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,- 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,- 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,- 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,- 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,- 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,- 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,- 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,- 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,- 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,- 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,- 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,- 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,- 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,- 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,- 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,- 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,- 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,- 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,- 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,- 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,-};--constexpr constant static uint64_t iq2s_grid[1024] = {+GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,@@ -4046,254 +947,4590 @@ constexpr constant static uint64_t iq2s_grid[1024] = {0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256)+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)+ 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,+ 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,+ 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,+ 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,+ 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,+ 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,+ 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,+ 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,+ 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,+ 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,+ 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,+ 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,+ 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,+ 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,+ 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,+ 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,+ 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,+ 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,+ 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,+ 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,+ 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,+ 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,+ 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,+ 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,+ 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,+ 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,+ 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,+ 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,+ 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,+ 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,+ 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,+ 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,+ 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,+ 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,+ 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,+ 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,+ 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,+ 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,+ 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,+ 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,+ 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,+ 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,+ 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,+ 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,+ 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,+ 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,+ 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,+ 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,+ 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,+ 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,+ 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,+ 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,+ 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,+ 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,+ 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,+ 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,+ 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,+ 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,+ 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,+ 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,+ 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,+ 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,+ 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,+ 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,+GGML_TABLE_END()++#define NGRID_IQ1S 2048+#define IQ1S_DELTA 0.125f+#if defined(GGML_COMMON_IMPL_C)+GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)+ 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,+ 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,+ 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000,+ 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000,+ 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101,+ 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff,+ 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00,+ 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101,+ 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100,+ 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00,+ 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000,+ 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000,+ 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101,+ 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff,+ 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100,+ 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01,+ 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000,+ 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff,+ 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001,+ 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000,+ 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00,+ 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff,+ 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff,+ 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000,+ 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff,+ 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff,+ 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101,+ 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff,+ 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000,+ 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100,+ 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01,+ 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00,+ 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00,+ 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01,+ 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff,+ 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000,+ 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff,+ 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000,+ 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101,+ 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100,+ 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff,+ 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff,+ 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001,+ 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01,+ 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff,+ 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000,+ 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000,+ 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff,+ 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01,+ 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00,+ 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000,+ 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000,+ 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000,+ 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001,+ 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff,+ 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01,+ 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff,+ 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff,+ 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000,+ 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000,+ 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00,+ 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001,+ 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100,+ 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101,+ 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00,+ 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00,+ 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000,+ 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001,+ 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff,+ 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000,+ 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000,+ 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff,+ 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100,+ 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000,+ 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101,+ 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001,+ 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000,+ 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff,+ 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01,+ 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100,+ 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000,+ 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01,+ 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101,+ 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000,+ 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff,+ 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff,+ 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001,+ 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001,+ 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000,+ 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000,+ 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00,+ 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100,+ 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01,+ 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00,+ 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff,+ 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000,+ 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00,+ 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000,+ 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff,+ 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00,+ 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000,+ 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000,+ 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff,+ 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00,+ 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00,+ 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001,+ 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001,+ 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000,+ 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100,+ 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101,+ 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101,+ 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000,+ 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00,+ 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff,+ 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000,+ 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff,+ 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff,+ 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff,+ 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101,+ 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001,+ 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100,+ 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101,+ 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01,+ 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00,+ 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00,+ 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000,+ 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101,+ 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100,+ 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001,+ 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff,+ 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00,+ 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000,+ 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100,+ 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00,+ 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01,+ 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff,+ 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101,+ 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100,+ 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100,+ 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000,+ 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff,+ 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff,+ 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000,+ 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101,+ 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000,+ 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101,+ 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101,+ 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100,+ 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01,+ 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001,+ 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001,+ 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff,+ 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff,+ 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000,+ 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000,+ 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101,+ 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff,+ 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001,+ 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000,+ 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff,+ 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00,+ 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff,+ 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000,+ 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00,+ 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff,+ 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000,+ 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000,+ 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff,+ 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000,+ 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000,+ 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000,+ 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101,+ 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff,+ 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100,+ 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101,+ 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001,+ 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff,+ 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100,+ 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff,+ 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01,+ 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff,+ 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff,+ 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff,+ 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff,+ 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001,+ 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff,+ 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01,+ 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00,+ 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100,+ 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101,+ 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff,+ 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00,+ 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01,+ 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00,+ 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100,+ 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00,+ 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000,+ 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000,+ 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000,+ 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000,+ 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001,+ 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff,+ 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00,+ 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff,+ 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00,+ 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000,+ 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff,+ 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff,+ 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101,+ 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000,+ 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff,+ 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff,+ 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff,+ 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff,+ 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000,+ 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000,+ 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100,+ 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00,+ 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100,+ 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100,+ 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00,+ 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff,+ 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff,+ 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100,+ 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff,+ 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000,+ 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00,+ 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,+ 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00,+ 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100,+ 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff,+ 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff,+ 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001,+ 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00,+ 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101,+ 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001,+ 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000,+ 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100,+ 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000,+ 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001,+ 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000,+ 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff,+ 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101,+ 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,+ 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101,+ 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000,+ 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff,+ 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000,+ 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff,+ 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01,+ 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100,+ 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff,+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101,+ 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001,+ 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01,+ 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff,+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,+ 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff,+ 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00,+ 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff,+ 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff,+ 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff,+ 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001,+ 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff,+ 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff,+ 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001,+ 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff,+ 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff,+ 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000,+ 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00,+ 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100,+ 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01,+ 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff,+ 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff,+ 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000,+ 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101,+ 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01,+ 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100,+ 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100,+ 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00,+ 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101,+ 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001,+ 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01,+ 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100,+ 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff,+ 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01,+ 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff,+ 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000,+ 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001,+ 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01,+ 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff,+ 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff,+ 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00,+ 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff,+ 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100,+ 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000,+ 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001,+ 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100,+ 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff,+ 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff,+ 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01,+ 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101,+ 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001,+ 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff,+ 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101,+ 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100,+ 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00,+ 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01,+ 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001,+ 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00,+ 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000,+ 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101,+ 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001,+ 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100,+ 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000,+ 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000,+ 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00,+ 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101,+ 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff,+ 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101,+ 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001,+ 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01,+ 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100,+ 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000,+ 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00,+ 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff,+ 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001,+ 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001,+ 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff,+ 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00,+ 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100,+ 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00,+ 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101,+ 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff,+ 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff,+ 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff,+ 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00,+ 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff,+ 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff,+ 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101,+ 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100,+ 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101,+ 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00,+ 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00,+ 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff,+ 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff,+ 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001,+ 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000,+ 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff,+ 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff,+ 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff,+ 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff,+ 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001,+ 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100,+ 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101,+ 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001,+ 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001,+ 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101,+ 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101,+ 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff,+ 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff,+ 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000,+ 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101,+ 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001,+ 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff,+ 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000,+ 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff,+ 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100,+ 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000,+ 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101,+ 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff,+ 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100,+ 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff,+ 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01,+ 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100,+ 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000,+ 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100,+ 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100,+ 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001,+ 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000,+ 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000,+ 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000,+ 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00,+ 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff,+ 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001,+ 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000,+ 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101,+ 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101,+ 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000,+ 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff,+ 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000,+ 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101,+ 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff,+ 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff,+ 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000,+ 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101,+ 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff,+ 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff,+ 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01,+ 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff,+ 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000,+ 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101,+ 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff,+ 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff,+ 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff,+ 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01,+ 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00,+ 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000,+ 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000,+ 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101,+ 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001,+ 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff,+ 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000,+ 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff,+ 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000,+ 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000,+ 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000,+ 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000,+ 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001,+ 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff,+ 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001,+ 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000,+ 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000,+ 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00,+ 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000,+ 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101,+ 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001,+ 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00,+ 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001,+ 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff,+ 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000,+ 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00,+ 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100,+ 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101,+ 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001,+ 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01,+ 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff,+ 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff,+ 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00,+ 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01,+ 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100,+ 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000,+ 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff,+ 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff,+ 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff,+ 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000,+ 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff,+ 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101,+ 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff,+ 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001,+ 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001,+ 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001,+ 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000,+ 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000,+ 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff,+ 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101,+ 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001,+ 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff,+ 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000,+ 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000,+ 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff,+ 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101,+ 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000,+ 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100,+ 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00,+ 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000,+ 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff,+ 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01,+ 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00,+ 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff,+ 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000,+ 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101,+ 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff,+ 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001,+ 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff,+ 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000,+ 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100,+ 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff,+ 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101,+ 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100,+ 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff,+ 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01,+ 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000,+ 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff,+ 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00,+ 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100,+ 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff,+ 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001,+ 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00,+ 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100,+ 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff,+ 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01,+ 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00,+ 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000,+ 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01,+ 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff,+ 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000,+ 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff,+ 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff,+ 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff,+ 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001,+ 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff,+ 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01,+ 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff,+ 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00,+ 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00,+ 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001,+ 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101,+ 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101,+ 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff,+ 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000,+ 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101,+GGML_TABLE_END()+#else+GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S)+ 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,+ 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,+ 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,+ 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,+ 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,+ 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,+ 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,+ 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,+ 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,+ 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,+ 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,+ 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,+ 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,+ 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,+ 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,+ 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,+ 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,+ 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,+ 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,+ 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,+ 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,+ 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,+ 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,+ 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,+ 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,+ 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,+ 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,+ 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,+ 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,+ 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,+ 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,+ 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,+ 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,+ 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,+ 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,+ 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,+ 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,+ 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,+ 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,+ 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,+ 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,+ 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,+ 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,+ 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,+ 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,+ 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,+ 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,+ 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,+ 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,+ 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,+ 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,+ 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,+ 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,+ 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,+ 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,+ 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,+ 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,+ 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,+ 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,+ 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,+ 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,+ 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,+ 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,+ 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,+ 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,+ 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,+ 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,+ 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,+ 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,+ 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,+ 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,+ 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,+ 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,+ 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,+ 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,+ 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,+ 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,+ 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,+ 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,+ 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,+ 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,+ 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,+ 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,+ 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,+ 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,+ 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,+ 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,+ 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,+ 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,+ 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,+ 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,+ 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,+ 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,+ 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,+ 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,+ 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,+ 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,+ 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,+ 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,+ 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,+ 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,+ 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,+ 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,+ 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,+ 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,+ 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,+ 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,+ 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,+ 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,+ 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,+ 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,+ 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,+ 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,+ 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,+ 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,+ 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,+ 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,+ 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,+ 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,+ 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,+ 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,+ 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,+ 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,+ 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,+ 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,+ 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,+ 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,+ 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,+ 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,+ 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,+ 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,+ 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,+ 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,+ 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,+ 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,+ 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,+ 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,+ 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,+ 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,+ 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,+ 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,+ 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,+ 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,+ 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,+ 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,+ 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,+ 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,+ 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,+ 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,+ 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,+ 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,+ 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,+ 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,+ 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,+ 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,+ 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,+ 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,+ 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,+ 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,+ 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,+ 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,+ 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,+ 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,+ 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,+ 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,+ 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,+ 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,+ 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,+ 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,+ 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,+ 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,+ 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,+ 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,+ 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,+ 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,+ 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,+ 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,+ 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,+ 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,+ 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,+ 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,+ 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,+ 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,+ 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,+ 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,+ 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,+ 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,+ 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,+ 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,+ 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,+ 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,+ 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,+ 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,+ 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,+ 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,+ 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,+ 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,+ 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,+ 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,+ 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,+ 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,+ 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,+ 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,+ 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,+ 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,+ 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,+ 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,+ 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,+ 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,+ 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,+ 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,+ 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,+ 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,+ 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,+ 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,+ 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,+ 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,+ 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,+ 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,+ 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,+ 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,+ 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,+ 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,+ 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,+ 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,+ 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,+ 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,+ 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,+ 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,+ 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,+ 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,+ 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,+ 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,+ 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,+ 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,+ 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,+ 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,+ 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,+ 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,+ 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,+ 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,+ 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,+ 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,+ 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,+ 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,+ 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,+ 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,+ 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,+ 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,+ 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,+ 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,+ 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,+ 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,+ 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,+ 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,+ 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,+GGML_TABLE_END()+#endif++#endif // GGML_COMMON_IMPL+#endif // GGML_COMMON_IMPL++#include <metal_stdlib>++using namespace metal;++#define MAX(x, y) ((x) > (y) ? (x) : (y))+#define MIN(x, y) ((x) < (y) ? (x) : (y))+#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }++#define N_SIMDWIDTH 32 // assuming SIMD group size is 32++enum ggml_sort_order {+ GGML_SORT_ASC,+ GGML_SORT_DESC,+};++// general-purpose kernel for addition, multiplication and division of two tensors+// pros: works for non-contiguous tensors, supports broadcast across all dims+// cons: not very efficient+kernel void kernel_add(+ device const char * src0,+ device const char * src1,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ constant int64_t & offs,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig.z;+ const int64_t i02 = tgpig.y;+ const int64_t i01 = tgpig.x;++ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;++ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + offs;+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + offs;++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ const int i10 = i0 % ne10;+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) + *((device float *)(src1_ptr + i10*nb10));+ }+}++kernel void kernel_mul(+ device const char * src0,+ device const char * src1,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig.z;+ const int64_t i02 = tgpig.y;+ const int64_t i01 = tgpig.x;++ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;++ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ const int i10 = i0 % ne10;+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) * *((device float *)(src1_ptr + i10*nb10));+ }+}++kernel void kernel_div(+ device const char * src0,+ device const char * src1,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig.z;+ const int64_t i02 = tgpig.y;+ const int64_t i01 = tgpig.x;++ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;++ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ const int i10 = i0 % ne10;+ *((device float *)(dst_ptr + i0*nb0)) = *((device float *)(src0_ptr + i0*nb00)) / *((device float *)(src1_ptr + i10*nb10));+ }+}++// assumption: src1 is a row+// broadcast src1 into src0+kernel void kernel_add_row(+ device const float4 * src0,+ device const float4 * src1,+ device float4 * dst,+ constant uint64_t & nb [[buffer(28)]],+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = src0[tpig] + src1[tpig % nb];+}++kernel void kernel_mul_row(+ device const float4 * src0,+ device const float4 * src1,+ device float4 * dst,+ constant uint64_t & nb [[buffer(28)]],+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = src0[tpig] * src1[tpig % nb];+}++kernel void kernel_div_row(+ device const float4 * src0,+ device const float4 * src1,+ device float4 * dst,+ constant uint64_t & nb [[buffer(28)]],+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = src0[tpig] / src1[tpig % nb];+}++kernel void kernel_scale(+ device const float * src0,+ device float * dst,+ constant float & scale,+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = src0[tpig] * scale;+}++kernel void kernel_scale_4(+ device const float4 * src0,+ device float4 * dst,+ constant float & scale,+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = src0[tpig] * scale;+}++kernel void kernel_relu(+ device const float * src0,+ device float * dst,+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = max(0.0f, src0[tpig]);+}++kernel void kernel_tanh(+ device const float * src0,+ device float * dst,+ uint tpig[[thread_position_in_grid]]) {+ device const float & x = src0[tpig];+ dst[tpig] = precise::tanh(x);+}++constant float GELU_COEF_A = 0.044715f;+constant float GELU_QUICK_COEF = -1.702f;+constant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;++kernel void kernel_gelu(+ device const float4 * src0,+ device float4 * dst,+ uint tpig[[thread_position_in_grid]]) {+ device const float4 & x = src0[tpig];++ // BEWARE !!!+ // Simply using "tanh" instead of "precise::tanh" will sometimes results in NaNs!+ // This was observed with Falcon 7B and 40B models+ //+ dst[tpig] = 0.5f*x*(1.0f + precise::tanh(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));+}++kernel void kernel_gelu_quick(+ device const float4 * src0,+ device float4 * dst,+ uint tpig[[thread_position_in_grid]]) {+ device const float4 & x = src0[tpig];++ dst[tpig] = x*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x)));+}++kernel void kernel_silu(+ device const float4 * src0,+ device float4 * dst,+ uint tpig[[thread_position_in_grid]]) {+ device const float4 & x = src0[tpig];+ dst[tpig] = x / (1.0f + exp(-x));+}++kernel void kernel_sqr(+ device const float * src0,+ device float * dst,+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = src0[tpig] * src0[tpig];+}++kernel void kernel_sum_rows(+ device const float * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tpig[[thread_position_in_grid]]) {+ int64_t i3 = tpig.z;+ int64_t i2 = tpig.y;+ int64_t i1 = tpig.x;++ if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {+ return;+ }++ device const float * src_row = (device const float *) ((device const char *) src0 + i1*nb01 + i2*nb02 + i3*nb03);+ device float * dst_row = (device float *) ((device char *) dst + i1*nb1 + i2*nb2 + i3*nb3);++ float row_sum = 0;++ for (int64_t i0 = 0; i0 < ne00; i0++) {+ row_sum += src_row[i0];+ }++ dst_row[0] = row_sum;+}++kernel void kernel_soft_max(+ device const float * src0,+ device const float * src1,+ device const float * src2,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant float & scale,+ constant float & max_bias,+ constant float & m0,+ constant float & m1,+ constant uint32_t & n_head_log2,+ threadgroup float * buf [[threadgroup(0)]],+ uint tgpig[[threadgroup_position_in_grid]],+ uint tpitg[[thread_position_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = (tgpig) / (ne02*ne01);+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);++ device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;+ device const float * pmask = src1 != src0 ? src1 + i01*ne00 : nullptr;+ device const float * ppos = src2 != src0 ? src2 : nullptr;+ device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ float slope = 0.0f;++ // ALiBi+ if (max_bias > 0.0f) {+ const int64_t h = i02;++ const float base = h < n_head_log2 ? m0 : m1;+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ slope = pow(base, exp);+ }++ // parallel max+ float lmax = -INFINITY;++ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {+ lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));+ }++ // find the max value in the block+ float max_val = simd_max(lmax);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = -INFINITY;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ buf[sgitg] = max_val;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ max_val = buf[tiisg];+ max_val = simd_max(max_val);+ }++ // parallel sum+ float lsum = 0.0f;+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {+ const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);+ lsum += exp_psrc0;+ pdst[i00] = exp_psrc0;+ }++ // This barrier fixes a failing test+ // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335+ threadgroup_barrier(mem_flags::mem_none);++ float sum = simd_sum(lsum);++ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ buf[sgitg] = sum;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ sum = buf[tiisg];+ sum = simd_sum(sum);+ }++ const float inv_sum = 1.0f/sum;++ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {+ pdst[i00] *= inv_sum;+ }+}++kernel void kernel_soft_max_4(+ device const float * src0,+ device const float * src1,+ device const float * src2,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant float & scale,+ constant float & max_bias,+ constant float & m0,+ constant float & m1,+ constant uint32_t & n_head_log2,+ threadgroup float * buf [[threadgroup(0)]],+ uint tgpig[[threadgroup_position_in_grid]],+ uint tpitg[[thread_position_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = (tgpig) / (ne02*ne01);+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);++ device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);+ device const float4 * pmask = src1 != src0 ? (device const float4 *)(src1 + i01*ne00) : nullptr;+ device const float4 * ppos = src2 != src0 ? (device const float4 *)(src2) : nullptr;+ device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);++ float slope = 0.0f;++ if (max_bias > 0.0f) {+ const int64_t h = i02;++ const float base = h < n_head_log2 ? m0 : m1;+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ slope = pow(base, exp);+ }++ // parallel max+ float4 lmax4 = -INFINITY;++ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));+ }++ const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));++ float max_val = simd_max(lmax);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = -INFINITY;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ buf[sgitg] = max_val;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ max_val = buf[tiisg];+ max_val = simd_max(max_val);+ }++ // parallel sum+ float4 lsum4 = 0.0f;+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);+ lsum4 += exp_psrc4;+ pdst4[i00] = exp_psrc4;+ }++ const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];++ // This barrier fixes a failing test+ // ref: https://github.com/ggerganov/ggml/pull/621#discussion_r1425156335+ threadgroup_barrier(mem_flags::mem_none);++ float sum = simd_sum(lsum);++ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ buf[sgitg] = sum;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ sum = buf[tiisg];+ sum = simd_sum(sum);+ }++ const float inv_sum = 1.0f/sum;++ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {+ pdst4[i00] *= inv_sum;+ }+}++kernel void kernel_diag_mask_inf(+ device const float * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int & n_past,+ uint3 tpig[[thread_position_in_grid]]) {+ const int64_t i02 = tpig[2];+ const int64_t i01 = tpig[1];+ const int64_t i00 = tpig[0];++ if (i00 > n_past + i01) {+ dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY;+ } else {+ dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00];+ }+}++kernel void kernel_diag_mask_inf_8(+ device const float4 * src0,+ device float4 * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int & n_past,+ uint3 tpig[[thread_position_in_grid]]) {++ const int64_t i = 2*tpig[0];++ dst[i+0] = src0[i+0];+ dst[i+1] = src0[i+1];+ int64_t i4 = 4*i;+ const int64_t i02 = i4/(ne00*ne01); i4 -= i02*ne00*ne01;+ const int64_t i01 = i4/(ne00); i4 -= i01*ne00;+ const int64_t i00 = i4;+ for (int k = 3; k >= 0; --k) {+ if (i00 + 4 + k <= n_past + i01) {+ break;+ }+ dst[i+1][k] = -INFINITY;+ if (i00 + k > n_past + i01) {+ dst[i][k] = -INFINITY;+ }+ }+}++kernel void kernel_norm(+ device const void * src0,+ device float * dst,+ constant int64_t & ne00,+ constant uint64_t & nb01,+ constant float & eps,+ threadgroup float * sum [[threadgroup(0)]],+ uint tgpig[[threadgroup_position_in_grid]],+ uint tpitg[[thread_position_in_threadgroup]],+ uint ntg[[threads_per_threadgroup]]) {+ device const float * x = (device const float *) ((device const char *) src0 + tgpig*nb01);+ // MEAN+ // parallel sum+ sum[tpitg] = 0.0f;+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {+ sum[tpitg] += x[i00];+ }+ // reduce+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint i = ntg/2; i > 0; i /= 2) {+ if (tpitg < i) {+ sum[tpitg] += sum[tpitg + i];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const float mean = sum[0] / ne00;++ // recenter and VARIANCE+ threadgroup_barrier(mem_flags::mem_threadgroup);+ device float * y = dst + tgpig*ne00;+ sum[tpitg] = 0.0f;+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {+ y[i00] = x[i00] - mean;+ sum[tpitg] += y[i00] * y[i00];+ }++ // reduce+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint i = ntg/2; i > 0; i /= 2) {+ if (tpitg < i) {+ sum[tpitg] += sum[tpitg + i];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const float variance = sum[0] / ne00;++ const float scale = 1.0f/sqrt(variance + eps);+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {+ y[i00] = y[i00] * scale;+ }+}++kernel void kernel_rms_norm(+ device const void * src0,+ device float * dst,+ constant int64_t & ne00,+ constant uint64_t & nb01,+ constant float & eps,+ threadgroup float * buf [[threadgroup(0)]],+ uint tgpig[[threadgroup_position_in_grid]],+ uint tpitg[[thread_position_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint ntg[[threads_per_threadgroup]]) {+ device const float4 * x = (device const float4 *) ((device const char *) src0 + tgpig*nb01);++ float4 sumf = 0;+ float all_sum = 0;++ // parallel sum+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {+ sumf += x[i00] * x[i00];+ }+ all_sum = sumf[0] + sumf[1] + sumf[2] + sumf[3];+ all_sum = simd_sum(all_sum);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ buf[sgitg] = all_sum;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ all_sum = buf[tiisg];+ all_sum = simd_sum(all_sum);+ }++ const float mean = all_sum/ne00;+ const float scale = 1.0f/sqrt(mean + eps);++ device float4 * y = (device float4 *) (dst + tgpig*ne00);+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {+ y[i00] = x[i00] * scale;+ }+}++kernel void kernel_group_norm(+ device const float * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int32_t & n_groups,+ constant float & eps,+ threadgroup float * buf [[threadgroup(0)]],+ uint tgpig[[threadgroup_position_in_grid]],+ uint tpitg[[thread_position_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint ntg[[threads_per_threadgroup]]) {+ const int64_t ne = ne00*ne01*ne02;+ const int64_t gs = ne00*ne01*((ne02 + n_groups - 1) / n_groups);++ int start = tgpig * gs;+ int end = start + gs;++ start += tpitg;++ if (end >= ne) {+ end = ne;+ }++ float tmp = 0.0f; // partial sum for thread in warp++ for (int j = start; j < end; j += ntg) {+ tmp += src0[j];+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);+ tmp = simd_sum(tmp);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ buf[sgitg] = tmp;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ tmp = buf[tiisg];+ tmp = simd_sum(tmp);+ }++ const float mean = tmp / gs;+ tmp = 0.0f;++ for (int j = start; j < end; j += ntg) {+ float xi = src0[j] - mean;+ dst[j] = xi;+ tmp += xi * xi;+ }++ tmp = simd_sum(tmp);+ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ buf[tiisg] = 0.0f;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ buf[sgitg] = tmp;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ tmp = buf[tiisg];+ tmp = simd_sum(tmp);+ }++ const float variance = tmp / gs;+ const float scale = 1.0f/sqrt(variance + eps);+ for (int j = start; j < end; j += ntg) {+ dst[j] *= scale;+ }+}++// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])+// il indicates where the q4 quants begin (0 or QK4_0/4)+// we assume that the yl's have been multiplied with the appropriate scale factor+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)+inline float block_q_n_dot_y(device const block_q4_0 * qb_curr, float sumy, thread float * yl, int il) {+ float d = qb_curr->d;++ float2 acc = 0.f;++ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 1 + il/2);++ for (int i = 0; i < 8; i+=2) {+ acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F)+ + yl[i + 1] * (qs[i / 2] & 0x0F00);+ acc[1] += yl[i + 8] * (qs[i / 2] & 0x00F0)+ + yl[i + 9] * (qs[i / 2] & 0xF000);+ }+ return d * (sumy * -8.f + acc[0] + acc[1]);+}++// function for calculate inner product between half a q4_1 block and 16 floats (yl), sumy is SUM(yl[i])+// il indicates where the q4 quants begin (0 or QK4_0/4)+// we assume that the yl's have been multiplied with the appropriate scale factor+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)+inline float block_q_n_dot_y(device const block_q4_1 * qb_curr, float sumy, thread float * yl, int il) {+ float d = qb_curr->d;+ float m = qb_curr->m;++ float2 acc = 0.f;++ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 2 + il/2);++ for (int i = 0; i < 8; i+=2) {+ acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F)+ + yl[i + 1] * (qs[i / 2] & 0x0F00);+ acc[1] += yl[i + 8] * (qs[i / 2] & 0x00F0)+ + yl[i + 9] * (qs[i / 2] & 0xF000);+ }+ return d * (acc[0] + acc[1]) + sumy * m;+}++// function for calculate inner product between half a q5_0 block and 16 floats (yl), sumy is SUM(yl[i])+// il indicates where the q5 quants begin (0 or QK5_0/4)+// we assume that the yl's have been multiplied with the appropriate scale factor+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)+inline float block_q_n_dot_y(device const block_q5_0 * qb_curr, float sumy, thread float * yl, int il) {+ float d = qb_curr->d;++ float2 acc = 0.f;++ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 3 + il/2);+ const uint32_t qh = *((device const uint32_t *)qb_curr->qh);++ for (int i = 0; i < 8; i+=2) {+ acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010))+ + yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));+ acc[1] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100))+ + yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));+ }+ return d * (sumy * -16.f + acc[0] + acc[1]);+}++// function for calculate inner product between half a q5_1 block and 16 floats (yl), sumy is SUM(yl[i])+// il indicates where the q5 quants begin (0 or QK5_1/4)+// we assume that the yl's have been multiplied with the appropriate scale factor+// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)+inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thread float * yl, int il) {+ float d = qb_curr->d;+ float m = qb_curr->m;++ float2 acc = 0.f;++ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 4 + il/2);+ const uint32_t qh = *((device const uint32_t *)qb_curr->qh);++ for (int i = 0; i < 8; i+=2) {+ acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010))+ + yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));+ acc[1] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100))+ + yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));+ }+ return d * (acc[0] + acc[1]) + sumy * m;+}++// putting them in the kernel cause a significant performance penalty+#define N_DST 4 // each SIMD group works on 4 rows+#define N_SIMDGROUP 2 // number of SIMD groups in a thread group+//Note: This is a template, but strictly speaking it only applies to+// quantizations where the block size is 32. It also does not+// guard against the number of rows not being divisible by+// N_DST, so this is another explicit assumption of the implementation.+template<typename block_q_type, int nr, int nsg, int nw>+void mul_vec_q_n_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ int64_t ne00,+ int64_t ne01,+ int64_t ne02,+ int64_t ne10,+ int64_t ne12,+ int64_t ne0,+ int64_t ne1,+ uint r2,+ uint r3,+ uint3 tgpig, uint tiisg, uint sgitg) {+ const int nb = ne00/QK4_0;++ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * nsg + sgitg) * nr;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q_type * x = (device const block_q_type *) src0 + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[16]; // src1 vector cache+ float sumf[nr] = {0.f};++ const int ix = (tiisg/2);+ const int il = (tiisg%2)*8;++ device const float * yb = y + ix * QK4_0 + il;++ // each thread in a SIMD group deals with half a block.+ for (int ib = ix; ib < nb; ib += nw/2) {+ float sumy = 0;+ for (int i = 0; i < 8; i += 2) {+ sumy += yb[i] + yb[i+1];+ yl[i+0] = yb[i+ 0];+ yl[i+1] = yb[i+ 1]/256.f;++ sumy += yb[i+16] + yb[i+17];+ yl[i+8] = yb[i+16]/16.f;+ yl[i+9] = yb[i+17]/4096.f;+ }++ for (int row = 0; row < nr; row++) {+ sumf[row] += block_q_n_dot_y(x+ib+row*nb, sumy, yl, il);+ }++ yb += QK4_0 * 16;+ }++ for (int row = 0; row < nr; ++row) {+ const float tot = simd_sum(sumf[row]);+ if (tiisg == 0 && first_row + row < ne01) {+ dst[im*ne0*ne1 + r1*ne0 + first_row + row] = tot;+ }+ }+}++kernel void kernel_mul_mv_q4_0_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ mul_vec_q_n_f32_impl<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);+}++kernel void kernel_mul_mv_q4_1_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ mul_vec_q_n_f32_impl<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);+}++kernel void kernel_mul_mv_q5_0_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ mul_vec_q_n_f32_impl<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);+}++kernel void kernel_mul_mv_q5_1_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ mul_vec_q_n_f32_impl<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);+}+++#define NB_Q8_0 8++void kernel_mul_mv_q8_0_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ const int nr = N_DST;+ const int nsg = N_SIMDGROUP;+ const int nw = N_SIMDWIDTH;++ const int nb = ne00/QK8_0;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * nsg + sgitg) * nr;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q8_0 * x = (device const block_q8_0 *) src0 + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[NB_Q8_0];+ float sumf[nr]={0.f};++ const int ix = tiisg/4;+ const int il = tiisg%4;++ device const float * yb = y + ix * QK8_0 + NB_Q8_0*il;++ // each thread in a SIMD group deals with NB_Q8_0 quants at a time+ for (int ib = ix; ib < nb; ib += nw/4) {+ for (int i = 0; i < NB_Q8_0; ++i) {+ yl[i] = yb[i];+ }++ for (int row = 0; row < nr; row++) {+ device const int8_t * qs = x[ib+row*nb].qs + NB_Q8_0*il;+ float sumq = 0.f;+ for (int iq = 0; iq < NB_Q8_0; ++iq) {+ sumq += qs[iq] * yl[iq];+ }+ sumf[row] += sumq*x[ib+row*nb].d;+ }++ yb += NB_Q8_0 * nw;+ }++ for (int row = 0; row < nr; ++row) {+ const float tot = simd_sum(sumf[row]);+ if (tiisg == 0 && first_row + row < ne01) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;+ }+ }+}++[[host_name("kernel_mul_mv_q8_0_f32")]]+kernel void kernel_mul_mv_q8_0_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ kernel_mul_mv_q8_0_f32_impl(src0,src1,dst,ne00,ne01,ne02,ne10,ne12,ne0,ne1,r2,r3,tgpig,tiisg,sgitg);+}++#define N_F32_F32 4++void kernel_mul_mv_f32_f32_impl(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {++ const int64_t r0 = tgpig.x;+ const int64_t rb = tgpig.y*N_F32_F32;+ const int64_t im = tgpig.z;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const float * x = (device const float *) (src0 + offset0);++ if (ne00 < 128) {+ for (int row = 0; row < N_F32_F32; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);++ float sumf = 0;+ for (int i = tiisg; i < ne00; i += 32) {+ sumf += (float) x[i] * (float) y[i];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ } else {+ device const float4 * x4 = (device const float4 *)x;+ for (int row = 0; row < N_F32_F32; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);+ device const float4 * y4 = (device const float4 *) y;++ float sumf = 0;+ for (int i = tiisg; i < ne00/4; i += 32) {+ for (int k = 0; k < 4; ++k) sumf += (float) x4[i][k] * y4[i][k];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) x[i] * y[i];+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ }+}++[[host_name("kernel_mul_mv_f32_f32")]]+kernel void kernel_mul_mv_f32_f32(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {+ kernel_mul_mv_f32_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);+}++#define N_F16_F16 4++kernel void kernel_mul_mv_f16_f16(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {++ const int64_t r0 = tgpig.x;+ const int64_t rb = tgpig.y*N_F16_F16;+ const int64_t im = tgpig.z;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half * x = (device const half *) (src0 + offset0);++ if (ne00 < 128) {+ for (int row = 0; row < N_F16_F16; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const half * y = (device const half *) (src1 + r1*nb11 + im*nb12);++ float sumf = 0;+ for (int i = tiisg; i < ne00; i += 32) {+ sumf += (half) x[i] * (half) y[i];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ } else {+ device const half4 * x4 = (device const half4 *)x;+ for (int row = 0; row < N_F16_F16; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const half * y = (device const half *) (src1 + r1*nb11 + im*nb12);+ device const half4 * y4 = (device const half4 *) y;++ float sumf = 0;+ for (int i = tiisg; i < ne00/4; i += 32) {+ for (int k = 0; k < 4; ++k) sumf += (half) x4[i][k] * y4[i][k];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (half) x[i] * y[i];+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ }+}++void kernel_mul_mv_f16_f32_1row_impl(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {++ const int64_t r0 = tgpig.x;+ const int64_t r1 = tgpig.y;+ const int64_t im = tgpig.z;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half * x = (device const half *) (src0 + offset0);+ device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);++ float sumf = 0;+ if (ne00 < 128) {+ for (int i = tiisg; i < ne00; i += 32) {+ sumf += (float) x[i] * (float) y[i];+ }+ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ } else {+ device const half4 * x4 = (device const half4 *) x;+ device const float4 * y4 = (device const float4 *) y;+ for (int i = tiisg; i < ne00/4; i += 32) {+ for (int k = 0; k < 4; ++k) sumf += (float)x4[i][k] * y4[i][k];+ }+ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) x[i] * y[i];+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+}++[[host_name("kernel_mul_mv_f16_f32_1row")]]+kernel void kernel_mul_mv_f16_f32_1row(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {+ kernel_mul_mv_f16_f32_1row_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);+}++#define N_F16_F32 4++void kernel_mul_mv_f16_f32_impl(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {++ const int64_t r0 = tgpig.x;+ const int64_t rb = tgpig.y*N_F16_F32;+ const int64_t im = tgpig.z;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half * x = (device const half *) (src0 + offset0);++ if (ne00 < 128) {+ for (int row = 0; row < N_F16_F32; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);++ float sumf = 0;+ for (int i = tiisg; i < ne00; i += 32) {+ sumf += (float) x[i] * (float) y[i];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ } else {+ device const half4 * x4 = (device const half4 *)x;+ for (int row = 0; row < N_F16_F32; ++row) {+ int r1 = rb + row;+ if (r1 >= ne11) {+ break;+ }++ device const float * y = (device const float *) (src1 + r1*nb11 + im*nb12);+ device const float4 * y4 = (device const float4 *) y;++ float sumf = 0;+ for (int i = tiisg; i < ne00/4; i += 32) {+ for (int k = 0; k < 4; ++k) sumf += (float) x4[i][k] * y4[i][k];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ for (int i = 4*(ne00/4); i < ne00; ++i) all_sum += (float) x[i] * y[i];+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+ }+}++[[host_name("kernel_mul_mv_f16_f32")]]+kernel void kernel_mul_mv_f16_f32(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {+ kernel_mul_mv_f16_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);+}++// Assumes row size (ne00) is a multiple of 4+kernel void kernel_mul_mv_f16_f32_l4(+ device const char * src0,+ device const char * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]]) {++ const int nrows = ne11;+ const int64_t r0 = tgpig.x;+ const int64_t im = tgpig.z;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb02*ne02;++ device const half4 * x4 = (device const half4 *) (src0 + offset0);++ for (int r1 = 0; r1 < nrows; ++r1) {+ device const float4 * y4 = (device const float4 *) (src1 + r1*nb11 + im*nb12);++ float sumf = 0;+ for (int i = tiisg; i < ne00/4; i += 32) {+ for (int k = 0; k < 4; ++k) sumf += (float) x4[i][k] * y4[i][k];+ }++ float all_sum = simd_sum(sumf);+ if (tiisg == 0) {+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;+ }+ }+}++kernel void kernel_alibi_f32(+ device const float * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ constant float & m0,+ constant float & m1,+ constant int & n_heads_log2_floor,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ //const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);++ const int64_t k = i3*ne3 + i2;++ float m_k;+ if (k < n_heads_log2_floor) {+ m_k = pow(m0, k + 1);+ } else {+ m_k = pow(m1, 2 * (k - n_heads_log2_floor) + 1);+ }++ device char * dst_row = (device char *) dst + i3*nb3 + i2*nb2 + i1*nb1;+ device const char * src_row = (device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01;+ for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {+ const float src_v = *(device float *)(src_row + i00*nb00);+ device float * dst_v = (device float *)(dst_row + i00*nb0);+ *dst_v = i00 * m_k + src_v;+ }+}++static float rope_yarn_ramp(const float low, const float high, const int i0) {+ const float y = (i0 / 2 - low) / max(0.001f, high - low);+ return 1.0f - min(1.0f, max(0.0f, y));+}++// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn+// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.+static void rope_yarn(+ float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,+ thread float * cos_theta, thread float * sin_theta+) {+ // Get n-d rotational scaling corrected for extrapolation+ float theta_interp = freq_scale * theta_extrap;+ float theta = theta_interp;+ if (ext_factor != 0.0f) {+ float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;++ // Get n-d magnitude scaling corrected for interpolation+ mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);+ }+ *cos_theta = cos(theta) * mscale;+ *sin_theta = sin(theta) * mscale;+}++// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get+// `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`+static float rope_yarn_corr_factor(int n_dims, int n_orig_ctx, float n_rot, float base) {+ return n_dims * log(n_orig_ctx / (n_rot * 2 * M_PI_F)) / (2 * log(base));+}++static void rope_yarn_corr_dims(+ int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]+) {+ // start and end correction dims+ dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_fast, freq_base)));+ dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_slow, freq_base)));+}++typedef void (rope_t)(+ device const void * src0,+ device const int32_t * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ constant int & n_past,+ constant int & n_dims,+ constant int & mode,+ constant int & n_orig_ctx,+ constant float & freq_base,+ constant float & freq_scale,+ constant float & ext_factor,+ constant float & attn_factor,+ constant float & beta_fast,+ constant float & beta_slow,+ uint tiitg[[thread_index_in_threadgroup]],+ uint3 tptg[[threads_per_threadgroup]],+ uint3 tgpig[[threadgroup_position_in_grid]]);++template<typename T>+kernel void kernel_rope(+ device const void * src0,+ device const int32_t * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ constant int & n_past,+ constant int & n_dims,+ constant int & mode,+ constant int & n_orig_ctx,+ constant float & freq_base,+ constant float & freq_scale,+ constant float & ext_factor,+ constant float & attn_factor,+ constant float & beta_fast,+ constant float & beta_slow,+ uint tiitg[[thread_index_in_threadgroup]],+ uint3 tptg[[threads_per_threadgroup]],+ uint3 tgpig[[threadgroup_position_in_grid]]) {+ const int64_t i3 = tgpig[2];+ const int64_t i2 = tgpig[1];+ const int64_t i1 = tgpig[0];++ const bool is_neox = mode & 2;++ float corr_dims[2];+ rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);++ device const int32_t * pos = src1;++ const int64_t p = pos[i2];++ const float theta_0 = (float)p;+ const float inv_ndims = -1.f/n_dims;++ if (!is_neox) {+ for (int64_t i0 = 2*tiitg; i0 < ne0; i0 += 2*tptg.x) {++ const float theta = theta_0 * pow(freq_base, inv_ndims*i0);+ float cos_theta, sin_theta;+ rope_yarn(theta, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);++ device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ const T x0 = src[0];+ const T x1 = src[1];++ dst_data[0] = x0*cos_theta - x1*sin_theta;+ dst_data[1] = x0*sin_theta + x1*cos_theta;+ }+ } else {+ for (int64_t ic = 2*tiitg; ic < ne0; ic += 2*tptg.x) {+ if (ic < n_dims) {+ const int64_t ib = 0;++ // simplified from `(ib * n_dims + ic) * inv_ndims`+ const float cur_rot = inv_ndims*ic - ib;++ const float theta = theta_0 * pow(freq_base, cur_rot);+ float cos_theta, sin_theta;+ rope_yarn(theta, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);++ const int64_t i0 = ib*n_dims + ic/2;++ device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ const float x0 = src[0];+ const float x1 = src[n_dims/2];++ dst_data[0] = x0*cos_theta - x1*sin_theta;+ dst_data[n_dims/2] = x0*sin_theta + x1*cos_theta;+ } else {+ const int64_t i0 = ic;++ device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ dst_data[0] = src[0];+ dst_data[1] = src[1];+ }+ }+ }+}++template [[host_name("kernel_rope_f32")]] kernel rope_t kernel_rope<float>;+template [[host_name("kernel_rope_f16")]] kernel rope_t kernel_rope<half>;++typedef void (im2col_t)(+ device const float * x,+ device char * dst,+ constant int32_t & ofs0,+ constant int32_t & ofs1,+ constant int32_t & IW,+ constant int32_t & IH,+ constant int32_t & CHW,+ constant int32_t & s0,+ constant int32_t & s1,+ constant int32_t & p0,+ constant int32_t & p1,+ constant int32_t & d0,+ constant int32_t & d1,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tgpg[[threadgroups_per_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]);++template <typename T>+kernel void kernel_im2col(+ device const float * x,+ device char * dst,+ constant int32_t & ofs0,+ constant int32_t & ofs1,+ constant int32_t & IW,+ constant int32_t & IH,+ constant int32_t & CHW,+ constant int32_t & s0,+ constant int32_t & s1,+ constant int32_t & p0,+ constant int32_t & p1,+ constant int32_t & d0,+ constant int32_t & d1,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tgpg[[threadgroups_per_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int32_t iiw = tgpig[2] * s0 + tpitg[2] * d0 - p0;+ const int32_t iih = tgpig[1] * s1 + tpitg[1] * d1 - p1;++ const int32_t offset_dst =+ (tpitg[0] * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * CHW ++ (tgpig[0] * (ntg[1] * ntg[2]) + tpitg[1] * ntg[2] + tpitg[2]);++ device T * pdst = (device T *) (dst);++ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ pdst[offset_dst] = 0.0f;+ } else {+ const int32_t offset_src = tpitg[0] * ofs0 + tgpig[0] * ofs1;+ pdst[offset_dst] = x[offset_src + iih * IW + iiw];+ }+}++template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col<float>;+template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col<half>;++kernel void kernel_upscale_f32(+ device const char * src0,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ constant int32_t & sf,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {++ const int64_t i3 = tgpig.z;+ const int64_t i2 = tgpig.y;+ const int64_t i1 = tgpig.x;++ const int64_t i03 = i3;+ const int64_t i02 = i2;+ const int64_t i01 = i1/sf;++ device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);+ device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1);++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ dst_ptr[i0] = src0_ptr[i0/sf];+ }+}++kernel void kernel_pad_f32(+ device const char * src0,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {++ const int64_t i3 = tgpig.z;+ const int64_t i2 = tgpig.y;+ const int64_t i1 = tgpig.x;++ const int64_t i03 = i3;+ const int64_t i02 = i2;+ const int64_t i01 = i1;++ device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);+ device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1);++ if (i1 < ne01 && i2 < ne02 && i3 < ne03) {+ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ if (i0 < ne00) {+ dst_ptr[i0] = src0_ptr[i0];+ } else {+ dst_ptr[i0] = 0.0f;+ }+ }++ return;+ }++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ dst_ptr[i0] = 0.0f;+ }+}++kernel void kernel_arange_f32(+ device char * dst,+ constant int64_t & ne0,+ constant float & start,+ constant float & step,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {++ device float * dst_ptr = (device float *) dst;++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ dst_ptr[i0] = start + step * i0;+ }+}++kernel void kernel_timestep_embedding_f32(+ device const char * src0,+ device char * dst,+ constant uint64_t & nb1,+ constant int & dim,+ constant int & max_period,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {++ int i = tgpig.x;+ device float * embed_data = (device float *)(dst + i*nb1);++ int half_ = dim / 2;+ for (int j = tpitg.x; j < half_; j += ntg.x) {+ float timestep = ((device float *)src0)[i];+ float freq = (float)exp(-log((float)max_period) * j / half_);+ float arg = timestep * freq;+ embed_data[j ] = cos(arg);+ embed_data[j + half_] = sin(arg);+ }++ if (dim % 2 != 0 && tpitg.x == 0) {+ embed_data[dim] = 0.f;+ }+}++// bitonic sort implementation following the CUDA kernels as reference+typedef void (argsort_t)(+ device const float * x,+ device int32_t * dst,+ constant int64_t & ncols,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]]);++template<ggml_sort_order order>+kernel void kernel_argsort_f32_i32(+ device const float * x,+ device int32_t * dst,+ constant int64_t & ncols,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]]) {+ // bitonic sort+ int col = tpitg[0];+ int row = tgpig[1];++ if (col >= ncols) return;++ device const float * x_row = x + row * ncols;+ device int32_t * dst_row = dst + row * ncols;++ // initialize indices+ if (col < ncols) {+ dst_row[col] = col;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (int k = 2; k <= ncols; k *= 2) {+ for (int j = k / 2; j > 0; j /= 2) {+ int ixj = col ^ j;+ if (ixj > col) {+ if ((col & k) == 0) {+ if (order == GGML_SORT_ASC ? x_row[dst_row[col]] > x_row[dst_row[ixj]] : x_row[dst_row[col]] < x_row[dst_row[ixj]]) {+ SWAP(dst_row[col], dst_row[ixj]);+ }+ } else {+ if (order == GGML_SORT_ASC ? x_row[dst_row[col]] < x_row[dst_row[ixj]] : x_row[dst_row[col]] > x_row[dst_row[ixj]]) {+ SWAP(dst_row[col], dst_row[ixj]);+ }+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }+}++template [[host_name("kernel_argsort_f32_i32_asc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_ASC>;+template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_argsort_f32_i32<GGML_SORT_DESC>;++kernel void kernel_leaky_relu_f32(+ device const float * src0,+ device float * dst,+ constant float & slope,+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = src0[tpig] > 0.0f ? src0[tpig] : src0[tpig] * slope;+}++kernel void kernel_cpy_f16_f16(+ device const half * src0,+ device half * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);++ device half * dst_data = (device half *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {+ device const half * src = (device half *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);+ dst_data[i00] = src[0];+ }+}++kernel void kernel_cpy_f16_f32(+ device const half * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);++ device float * dst_data = (device float *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {+ device const half * src = (device half *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);+ dst_data[i00] = src[0];+ }+}++kernel void kernel_cpy_f32_f16(+ device const float * src0,+ device half * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);++ device half * dst_data = (device half *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ dst_data[i00] = src[0];+ }+}++kernel void kernel_cpy_f32_f32(+ device const float * src0,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);++ device float * dst_data = (device float *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ dst_data[i00] = src[0];+ }+}++kernel void kernel_cpy_f32_q8_0(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK8_0;++ device block_q8_0 * dst_data = (device block_q8_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK8_0; i00 < ne00; i00 += ntg.x*QK8_0) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK8_0; j++) {+ const float v = src[j];+ amax = MAX(amax, fabs(v));+ }++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK8_0].d = d;++ for (int j = 0; j < QK8_0; ++j) {+ const float x0 = src[j]*id;++ dst_data[i00/QK8_0].qs[j] = round(x0);+ }+ }+}++kernel void kernel_cpy_f32_q4_0(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_0;++ device block_q4_0 * dst_data = (device block_q4_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK4_0; i00 < ne00; i00 += ntg.x*QK4_0) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float amax = 0.0f; // absolute max+ float max = 0.0f;++ for (int j = 0; j < QK4_0; j++) {+ const float v = src[j];+ if (amax < fabs(v)) {+ amax = fabs(v);+ max = v;+ }+ }++ const float d = max / -8;+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK4_0].d = d;++ for (int j = 0; j < QK4_0/2; ++j) {+ const float x0 = src[0 + j]*id;+ const float x1 = src[QK4_0/2 + j]*id;++ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f));+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f));++ dst_data[i00/QK4_0].qs[j] = xi0;+ dst_data[i00/QK4_0].qs[j] |= xi1 << 4;+ }+ }+}++kernel void kernel_cpy_f32_q4_1(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_1;++ device block_q4_1 * dst_data = (device block_q4_1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK4_1; i00 < ne00; i00 += ntg.x*QK4_1) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float min = FLT_MAX;+ float max = -FLT_MAX;++ for (int j = 0; j < QK4_1; j++) {+ const float v = src[j];+ if (min > v) min = v;+ if (max < v) max = v;+ }++ const float d = (max - min) / ((1 << 4) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK4_1].d = d;+ dst_data[i00/QK4_1].m = min;++ for (int j = 0; j < QK4_1/2; ++j) {+ const float x0 = (src[0 + j] - min)*id;+ const float x1 = (src[QK4_1/2 + j] - min)*id;++ const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f));+ const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f));++ dst_data[i00/QK4_1].qs[j] = xi0;+ dst_data[i00/QK4_1].qs[j] |= xi1 << 4;+ }+ }+}++kernel void kernel_cpy_f32_q5_0(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK5_0;++ device block_q5_0 * dst_data = (device block_q5_0 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK5_0; i00 < ne00; i00 += ntg.x*QK5_0) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float amax = 0.0f; // absolute max+ float max = 0.0f;++ for (int j = 0; j < QK5_0; j++) {+ const float v = src[j];+ if (amax < fabs(v)) {+ amax = fabs(v);+ max = v;+ }+ }++ const float d = max / -16;+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK5_0].d = d;++ uint32_t qh = 0;+ for (int j = 0; j < QK5_0/2; ++j) {+ const float x0 = src[0 + j]*id;+ const float x1 = src[QK5_0/2 + j]*id;++ const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f));+ const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f));++ dst_data[i00/QK5_0].qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);+ }+ thread const uint8_t * qh8 = (thread const uint8_t *)&qh;+ for (int j = 0; j < 4; ++j) {+ dst_data[i00/QK5_0].qh[j] = qh8[j];+ }+ }+}++kernel void kernel_cpy_f32_q5_1(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK5_1;++ device block_q5_1 * dst_data = (device block_q5_1 *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK5_1; i00 < ne00; i00 += ntg.x*QK5_1) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float max = src[0];+ float min = src[0];++ for (int j = 1; j < QK5_1; j++) {+ const float v = src[j];+ min = v < min ? v : min;+ max = v > max ? v : max;+ }++ const float d = (max - min) / 31;+ const float id = d ? 1.0f/d : 0.0f;++ dst_data[i00/QK5_1].d = d;+ dst_data[i00/QK5_1].m = min;++ uint32_t qh = 0;+ for (int j = 0; j < QK5_1/2; ++j) {+ const float x0 = (src[0 + j] - min)*id;+ const float x1 = (src[QK5_1/2 + j] - min)*id;++ const uint8_t xi0 = (uint8_t)(x0 + 0.5f);+ const uint8_t xi1 = (uint8_t)(x1 + 0.5f);++ dst_data[i00/QK5_1].qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);+ }+ thread const uint8_t * qh8 = (thread const uint8_t *)&qh;+ for (int j = 0; j < 4; ++j) {+ dst_data[i00/QK5_1].qh[j] = qh8[j];+ }+ }+}++static inline int best_index_int8(int n, constant float * val, float x) {+ if (x <= val[0]) return 0;+ if (x >= val[n-1]) return n-1;+ int ml = 0, mu = n-1;+ while (mu-ml > 1) {+ int mav = (ml+mu)/2;+ if (x < val[mav]) mu = mav; else ml = mav;+ }+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;+}++constexpr constant static float kvalues_iq4nl_f[16] = {+ -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f};-constexpr constant static uint32_t iq3xxs_grid[256] = {- 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,- 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,- 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,- 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,- 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,- 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,- 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,- 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,- 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,- 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,- 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,- 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,- 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,- 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,- 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,- 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,- 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,- 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,- 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,- 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,- 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,- 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,- 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,- 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,- 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,- 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,- 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,- 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,- 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,- 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,- 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,- 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,-};+kernel void kernel_cpy_f32_iq4_nl(+ device const float * src0,+ device void * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {+ const int64_t i03 = tgpig[2];+ const int64_t i02 = tgpig[1];+ const int64_t i01 = tgpig[0];++ const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;++ const int64_t i3 = n / (ne2*ne1*ne0);+ const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);+ const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;+ const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0)/QK4_NL;++ device block_iq4_nl * dst_data = (device block_iq4_nl *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ for (int64_t i00 = tpitg.x*QK4_NL; i00 < ne00; i00 += ntg.x*QK4_NL) {+ device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);++ float amax = 0.0f; // absolute max+ float max = 0.0f;++ for (int j = 0; j < QK4_0; j++) {+ const float v = src[j];+ if (amax < fabs(v)) {+ amax = fabs(v);+ max = v;+ }+ }++ const float d = max / kvalues_iq4nl_f[0];+ const float id = d ? 1.0f/d : 0.0f;++ float sumqx = 0, sumq2 = 0;+ for (int j = 0; j < QK4_NL/2; ++j) {+ const float x0 = src[0 + j]*id;+ const float x1 = src[QK4_NL/2 + j]*id;++ const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl_f, x0);+ const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl_f, x1);++ dst_data[i00/QK4_NL].qs[j] = xi0 | (xi1 << 4);++ const float v0 = kvalues_iq4nl_f[xi0];+ const float v1 = kvalues_iq4nl_f[xi1];+ const float w0 = src[0 + j]*src[0 + j];+ const float w1 = src[QK4_NL/2 + j]*src[QK4_NL/2 + j];+ sumqx += w0*v0*src[j] + w1*v1*src[QK4_NL/2 + j];+ sumq2 += w0*v0*v0 + w1*v1*v1;++ }++ dst_data[i00/QK4_NL].d = sumq2 > 0 ? sumqx/sumq2 : d;++ }+}++kernel void kernel_concat(+ device const char * src0,+ device const char * src1,+ device char * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant int64_t & ne2,+ constant int64_t & ne3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint3 tpitg[[thread_position_in_threadgroup]],+ uint3 ntg[[threads_per_threadgroup]]) {++ const int64_t i03 = tgpig.z;+ const int64_t i02 = tgpig.y;+ const int64_t i01 = tgpig.x;++ const int64_t i13 = i03 % ne13;+ const int64_t i12 = i02 % ne12;+ const int64_t i11 = i01 % ne11;++ device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + tpitg.x*nb00;+ device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11 + tpitg.x*nb10;+ device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + tpitg.x*nb0;++ for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {+ if (i02 < ne02) {+ ((device float *)dst_ptr)[0] = ((device float *)src0_ptr)[0];+ src0_ptr += ntg.x*nb00;+ } else {+ ((device float *)dst_ptr)[0] = ((device float *)src1_ptr)[0];+ src1_ptr += ntg.x*nb10;+ }+ dst_ptr += ntg.x*nb0;+ }+}++void kernel_mul_mv_q2_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int nb = ne00/QK_K;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;+ const int ib_row = first_row * nb;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q2_K * x = (device const block_q2_K *) src0 + ib_row + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[32];+ float sumf[N_DST]={0.f}, all_sum;++ const int step = sizeof(block_q2_K) * nb;++#if QK_K == 256+ const int ix = tiisg/8; // 0...3+ const int it = tiisg%8; // 0...7+ const int iq = it/4; // 0 or 1+ const int ir = it%4; // 0...3+ const int is = (8*ir)/16;// 0 or 1++ device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;++ for (int ib = ix; ib < nb; ib += 4) {++ float4 sumy = {0.f, 0.f, 0.f, 0.f};+ for (int i = 0; i < 8; ++i) {+ yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];+ yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8];+ yl[i+16] = y4[i+64]; sumy[2] += yl[i+16];+ yl[i+24] = y4[i+96]; sumy[3] += yl[i+24];+ }++ device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is;+ device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;+ device const half * dh = &x[ib].d;++ for (int row = 0; row < N_DST; row++) {++ float4 acc1 = {0.f, 0.f, 0.f, 0.f};+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};+ for (int i = 0; i < 8; i += 2) {+ acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);+ acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);+ acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);+ acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);+ acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);+ acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);+ acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);+ acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);+ }+ float dall = dh[0];+ float dmin = dh[1] * 1.f/16.f;+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f ++ (acc1[1] + 1.f/256.f * acc2[1]) * (sc[2] & 0xF) * 1.f/ 4.f ++ (acc1[2] + 1.f/256.f * acc2[2]) * (sc[4] & 0xF) * 1.f/16.f ++ (acc1[3] + 1.f/256.f * acc2[3]) * (sc[6] & 0xF) * 1.f/64.f) -+ dmin * (sumy[0] * (sc[0] & 0xF0) + sumy[1] * (sc[2] & 0xF0) + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0));++ qs += step/2;+ sc += step;+ dh += step/2;+ }++ y4 += 4 * QK_K;+ }+#else+ const int ix = tiisg/2; // 0...15+ const int it = tiisg%2; // 0...1++ device const float * y4 = y + ix * QK_K + 8 * it;++ for (int ib = ix; ib < nb; ib += 16) {++ float4 sumy = {0.f, 0.f, 0.f, 0.f};+ for (int i = 0; i < 8; ++i) {+ yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];+ yl[i+ 8] = y4[i+16]; sumy[1] += yl[i+ 8];+ yl[i+16] = y4[i+32]; sumy[2] += yl[i+16];+ yl[i+24] = y4[i+48]; sumy[3] += yl[i+24];+ }++ device const uint8_t * sc = (device const uint8_t *)x[ib].scales;+ device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 4 * it;+ device const half * dh = &x[ib].d;++ for (int row = 0; row < N_DST; row++) {++ float4 acc1 = {0.f, 0.f, 0.f, 0.f};+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};+ for (int i = 0; i < 8; i += 2) {+ acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);+ acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);+ acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);+ acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);+ acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);+ acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);+ acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);+ acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);+ }++ float dall = dh[0];+ float dmin = dh[1];+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f ++ (acc1[1] + 1.f/256.f * acc2[1]) * (sc[1] & 0xF) * 1.f/ 4.f ++ (acc1[2] + 1.f/256.f * acc2[2]) * (sc[2] & 0xF) * 1.f/16.f ++ (acc1[3] + 1.f/256.f * acc2[3]) * (sc[3] & 0xF) * 1.f/64.f) -+ dmin * (sumy[0] * (sc[0] >> 4) + sumy[1] * (sc[1] >> 4) + sumy[2] * (sc[2] >> 4) + sumy[3] * (sc[3] >> 4));++ qs += step/2;+ sc += step;+ dh += step/2;+ }++ y4 += 16 * QK_K;+ }+#endif++ for (int row = 0; row < N_DST; ++row) {+ all_sum = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;+ }+ }+}++[[host_name("kernel_mul_mv_q2_K_f32")]]+kernel void kernel_mul_mv_q2_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q2_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}++#if QK_K == 256+void kernel_mul_mv_q3_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int nb = ne00/QK_K;++ const int64_t r0 = tgpig.x;+ const int64_t r1 = tgpig.y;+ const int64_t im = tgpig.z;++ const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q3_K * x = (device const block_q3_K *) src0 + first_row*nb + offset0;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[32];++ //const uint16_t kmask1 = 0x3030;+ //const uint16_t kmask2 = 0x0f0f;++ const int tid = tiisg/4;+ const int ix = tiisg%4;+ const int ip = tid/4; // 0 or 1+ const int il = 2*((tid%4)/2); // 0 or 2+ const int ir = tid%2;+ const int n = 8;+ const int l0 = n*ir;++ // One would think that the Metal compiler would figure out that ip and il can only have+ // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it+ // with these two tales.+ //+ // Possible masks for the high bit+ const ushort4 mm[4] = {{0x0001, 0x0100, 0x0002, 0x0200}, // ip = 0, il = 0+ {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2+ {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0+ {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2++ // Possible masks for the low 2 bits+ const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}};++ const ushort4 hm = mm[2*ip + il/2];++ const int shift = 2*il;+ const float v1 = il == 0 ? 4.f : 64.f;+ const float v2 = 4.f * v1;++ const uint16_t s_shift1 = 4*ip;+ const uint16_t s_shift2 = s_shift1 + il;++ const int q_offset = 32*ip + l0;+ const int y_offset = 128*ip + 32*il + l0;++ const int step = sizeof(block_q3_K) * nb / 2;++ device const float * y1 = yy + ix*QK_K + y_offset;++ uint32_t scales32, aux32;+ thread uint16_t * scales16 = (thread uint16_t *)&scales32;+ thread const int8_t * scales = (thread const int8_t *)&scales32;++ float sumf1[2] = {0.f};+ float sumf2[2] = {0.f};+ for (int i = ix; i < nb; i += 4) {++ for (int l = 0; l < 8; ++l) {+ yl[l+ 0] = y1[l+ 0];+ yl[l+ 8] = y1[l+16];+ yl[l+16] = y1[l+32];+ yl[l+24] = y1[l+48];+ }++ device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset);+ device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0);+ device const uint16_t * a = (device const uint16_t *)(x[i].scales);+ device const half * dh = &x[i].d;++ for (int row = 0; row < 2; ++row) {++ const float d_all = (float)dh[0];++ scales16[0] = a[4];+ scales16[1] = a[5];+ aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030;+ scales16[0] = a[il+0];+ scales16[1] = a[il+1];+ scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32;++ float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;+ for (int l = 0; l < n; l += 2) {+ const int32_t qs = q[l/2];+ s1 += yl[l+0] * (qs & qm[il/2][0]);+ s2 += yl[l+1] * (qs & qm[il/2][1]);+ s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]);+ s4 += yl[l+16] * (qs & qm[il/2][2]);+ s5 += yl[l+17] * (qs & qm[il/2][3]);+ s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]);+ }+ float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);+ float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);+ sumf1[row] += d1 * (scales[0] - 32);+ sumf2[row] += d2 * (scales[2] - 32);++ s1 = s2 = s3 = s4 = s5 = s6 = 0;+ for (int l = 0; l < n; l += 2) {+ const int32_t qs = q[l/2+8];+ s1 += yl[l+8] * (qs & qm[il/2][0]);+ s2 += yl[l+9] * (qs & qm[il/2][1]);+ s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]);+ s4 += yl[l+24] * (qs & qm[il/2][2]);+ s5 += yl[l+25] * (qs & qm[il/2][3]);+ s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]);+ }+ d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);+ d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);+ sumf1[row] += d1 * (scales[1] - 32);+ sumf2[row] += d2 * (scales[3] - 32);++ q += step;+ h += step;+ a += step;+ dh += step;++ }++ y1 += 4 * QK_K;++ }++ for (int row = 0; row < 2; ++row) {+ const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift);+ sumf1[row] = simd_sum(sumf);+ }+ if (tiisg == 0) {+ for (int row = 0; row < 2; ++row) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = sumf1[row];+ }+ }+}+#else+void kernel_mul_mv_q3_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int nb = ne00/QK_K;++ const int64_t r0 = tgpig.x;+ const int64_t r1 = tgpig.y;+ const int64_t im = tgpig.z;++ const int row = 2 * r0 + sgitg;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q3_K * x = (device const block_q3_K *) src0 + row*nb + offset0;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ const int ix = tiisg/4;+ const int il = 4 * (tiisg%4);// 0, 4, 8, 12+ const int iq = il/8; // 0, 0, 1, 1+ const int in = il%8; // 0, 4, 0, 4++ float2 sum = {0.f, 0.f};++ for (int i = ix; i < nb; i += 8) {++ const float d_all = (float)(x[i].d);++ device const uint16_t * q = (device const uint16_t *)(x[i].qs + il);+ device const uint16_t * h = (device const uint16_t *)(x[i].hmask + in);+ device const uint16_t * s = (device const uint16_t *)(x[i].scales);+ device const float * y = yy + i * QK_K + il;++ const float d1 = d_all * ((int32_t)(s[0] & 0x000F) - 8);+ const float d2 = d_all * ((int32_t)(s[0] & 0x00F0) - 128) * 1.f/64.f;+ const float d3 = d_all * ((int32_t)(s[0] & 0x0F00) - 2048) * 1.f/4096.f;+ const float d4 = d_all * ((int32_t)(s[0] & 0xF000) - 32768) * 1.f/262144.f;++ for (int l = 0; l < 4; l += 2) {+ const uint16_t hm = h[l/2] >> iq;+ sum[0] += y[l+ 0] * d1 * ((int32_t)(q[l/2] & 0x0003) - ((hm & 0x0001) ? 0 : 4))+ + y[l+16] * d2 * ((int32_t)(q[l/2] & 0x000c) - ((hm & 0x0004) ? 0 : 16))+ + y[l+32] * d3 * ((int32_t)(q[l/2] & 0x0030) - ((hm & 0x0010) ? 0 : 64))+ + y[l+48] * d4 * ((int32_t)(q[l/2] & 0x00c0) - ((hm & 0x0040) ? 0 : 256));+ sum[1] += y[l+ 1] * d1 * ((int32_t)(q[l/2] & 0x0300) - ((hm & 0x0100) ? 0 : 1024))+ + y[l+17] * d2 * ((int32_t)(q[l/2] & 0x0c00) - ((hm & 0x0400) ? 0 : 4096))+ + y[l+33] * d3 * ((int32_t)(q[l/2] & 0x3000) - ((hm & 0x1000) ? 0 : 16384))+ + y[l+49] * d4 * ((int32_t)(q[l/2] & 0xc000) - ((hm & 0x4000) ? 0 : 65536));+ }++ }+ const float sumf = sum[0] + sum[1] * 1.f/256.f;++ const float tot = simd_sum(sumf);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + row] = tot;+ }++}+#endif++[[host_name("kernel_mul_mv_q3_K_f32")]]+kernel void kernel_mul_mv_q3_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q3_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}++#if QK_K == 256+void kernel_mul_mv_q4_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const uint16_t kmask1 = 0x3f3f;+ const uint16_t kmask2 = 0x0f0f;+ const uint16_t kmask3 = 0xc0c0;++ const int ix = tiisg/8; // 0...3+ const int it = tiisg%8; // 0...7+ const int iq = it/4; // 0 or 1+ const int ir = it%4; // 0...3++ const int nb = ne00/QK_K;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;+ //const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;+ const int first_row = r0 * N_DST;+ const int ib_row = first_row * nb;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[16];+ float yh[16];+ float sumf[N_DST]={0.f}, all_sum;++ const int step = sizeof(block_q4_K) * nb / 2;++ device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;++ uint16_t sc16[4];+ thread const uint8_t * sc8 = (thread const uint8_t *)sc16;++ for (int ib = ix; ib < nb; ib += 4) {++ float4 sumy = {0.f, 0.f, 0.f, 0.f};+ for (int i = 0; i < 8; ++i) {+ yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0];+ yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8];+ yh[i+0] = y4[i+128]; sumy[2] += yh[i+0];+ yh[i+8] = y4[i+160]; sumy[3] += yh[i+8];+ }++ device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq;+ device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;+ device const half * dh = &x[ib].d;++ for (int row = 0; row < N_DST; row++) {++ sc16[0] = sc[0] & kmask1;+ sc16[1] = sc[2] & kmask1;+ sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);+ sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);++ device const uint16_t * q2 = q1 + 32;++ float4 acc1 = {0.f, 0.f, 0.f, 0.f};+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};+ for (int i = 0; i < 8; i += 2) {+ acc1[0] += yl[i+0] * (q1[i/2] & 0x000F);+ acc1[1] += yl[i+1] * (q1[i/2] & 0x0F00);+ acc1[2] += yl[i+8] * (q1[i/2] & 0x00F0);+ acc1[3] += yl[i+9] * (q1[i/2] & 0xF000);+ acc2[0] += yh[i+0] * (q2[i/2] & 0x000F);+ acc2[1] += yh[i+1] * (q2[i/2] & 0x0F00);+ acc2[2] += yh[i+8] * (q2[i/2] & 0x00F0);+ acc2[3] += yh[i+9] * (q2[i/2] & 0xF000);+ }++ float dall = dh[0];+ float dmin = dh[1];+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] ++ (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f ++ (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] ++ (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) -+ dmin * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);++ q1 += step;+ sc += step;+ dh += step;+ }++ y4 += 4 * QK_K;+ }++ for (int row = 0; row < N_DST; ++row) {+ all_sum = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;+ }+ }+}+#else+void kernel_mul_mv_q4_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int ix = tiisg/4; // 0...7+ const int it = tiisg%4; // 0...3++ const int nb = ne00/QK_K;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;+ const int first_row = r0 * N_DST;+ const int ib_row = first_row * nb;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[8];+ float yh[8];+ float sumf[N_DST]={0.f}, all_sum;++ const int step = sizeof(block_q4_K) * nb / 2;++ device const float * y4 = y + ix * QK_K + 8 * it;++ uint16_t sc16[4];++ for (int ib = ix; ib < nb; ib += 8) {++ float2 sumy = {0.f, 0.f};+ for (int i = 0; i < 8; ++i) {+ yl[i] = y4[i+ 0]; sumy[0] += yl[i];+ yh[i] = y4[i+32]; sumy[1] += yh[i];+ }++ device const uint16_t * sc = (device const uint16_t *)x[ib].scales;+ device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 4 * it;+ device const half * dh = x[ib].d;++ for (int row = 0; row < N_DST; row++) {++ sc16[0] = sc[0] & 0x000f;+ sc16[1] = sc[0] & 0x0f00;+ sc16[2] = sc[0] & 0x00f0;+ sc16[3] = sc[0] & 0xf000;++ float2 acc1 = {0.f, 0.f};+ float2 acc2 = {0.f, 0.f};+ for (int i = 0; i < 8; i += 2) {+ acc1[0] += yl[i+0] * (qs[i/2] & 0x000F);+ acc1[1] += yl[i+1] * (qs[i/2] & 0x0F00);+ acc2[0] += yh[i+0] * (qs[i/2] & 0x00F0);+ acc2[1] += yh[i+1] * (qs[i/2] & 0xF000);+ }++ float dall = dh[0];+ float dmin = dh[1];+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc1[1]) * sc16[0] ++ (acc2[0] + 1.f/256.f * acc2[1]) * sc16[1] * 1.f/4096.f) -+ dmin * 1.f/16.f * (sumy[0] * sc16[2] + sumy[1] * sc16[3] * 1.f/256.f);++ qs += step;+ sc += step;+ dh += step;+ }++ y4 += 8 * QK_K;+ }++ for (int row = 0; row < N_DST; ++row) {+ all_sum = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;+ }+ }+}+#endif++[[host_name("kernel_mul_mv_q4_K_f32")]]+kernel void kernel_mul_mv_q4_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q4_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}++void kernel_mul_mv_q5_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int nb = ne00/QK_K;++ const int64_t r0 = tgpig.x;+ const int64_t r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * N_SIMDGROUP + sgitg) * 2;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q5_K * x = (device const block_q5_K *) src0 + first_row*nb + offset0;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float sumf[2]={0.f};++ const int step = sizeof(block_q5_K) * nb;++#if QK_K == 256+#+ float yl[16], yh[16];++ const uint16_t kmask1 = 0x3f3f;+ const uint16_t kmask2 = 0x0f0f;+ const uint16_t kmask3 = 0xc0c0;++ const int tid = tiisg/4;+ const int ix = tiisg%4;+ const int iq = tid/4;+ const int ir = tid%4;+ const int n = 8;++ const int l0 = n*ir;+ const int q_offset = 32*iq + l0;+ const int y_offset = 64*iq + l0;++ const uint8_t hm1 = 1u << (2*iq);+ const uint8_t hm2 = hm1 << 1;+ const uint8_t hm3 = hm1 << 4;+ const uint8_t hm4 = hm2 << 4;++ uint16_t sc16[4];+ thread const uint8_t * sc8 = (thread const uint8_t *)sc16;++ device const float * y1 = yy + ix*QK_K + y_offset;++ for (int i = ix; i < nb; i += 4) {++ device const uint8_t * q1 = x[i].qs + q_offset;+ device const uint8_t * qh = x[i].qh + l0;+ device const half * dh = &x[i].d;+ device const uint16_t * a = (device const uint16_t *)x[i].scales + iq;++ device const float * y2 = y1 + 128;+ float4 sumy = {0.f, 0.f, 0.f, 0.f};+ for (int l = 0; l < 8; ++l) {+ yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0];+ yl[l+8] = y1[l+32]; sumy[1] += yl[l+8];+ yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0];+ yh[l+8] = y2[l+32]; sumy[3] += yh[l+8];+ }++ for (int row = 0; row < 2; ++row) {++ device const uint8_t * q2 = q1 + 64;++ sc16[0] = a[0] & kmask1;+ sc16[1] = a[2] & kmask1;+ sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2);+ sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2);++ float4 acc1 = {0.f};+ float4 acc2 = {0.f};+ for (int l = 0; l < n; ++l) {+ uint8_t h = qh[l];+ acc1[0] += yl[l+0] * (q1[l] & 0x0F);+ acc1[1] += yl[l+8] * (q1[l] & 0xF0);+ acc1[2] += yh[l+0] * (q2[l] & 0x0F);+ acc1[3] += yh[l+8] * (q2[l] & 0xF0);+ acc2[0] += h & hm1 ? yl[l+0] : 0.f;+ acc2[1] += h & hm2 ? yl[l+8] : 0.f;+ acc2[2] += h & hm3 ? yh[l+0] : 0.f;+ acc2[3] += h & hm4 ? yh[l+8] : 0.f;+ }+ const float dall = dh[0];+ const float dmin = dh[1];+ sumf[row] += dall * (sc8[0] * (acc1[0] + 16.f*acc2[0]) ++ sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) ++ sc8[4] * (acc1[2] + 16.f*acc2[2]) ++ sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) -+ dmin * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);++ q1 += step;+ qh += step;+ dh += step/2;+ a += step/2;++ }++ y1 += 4 * QK_K;++ }+#else+ float yl[8], yh[8];++ const int il = 4 * (tiisg/8); // 0, 4, 8, 12+ const int ix = tiisg%8;+ const int iq = il/8; // 0, 0, 1, 1+ const int in = il%8; // 0, 4, 0, 4++ device const float * y = yy + ix*QK_K + il;++ for (int i = ix; i < nb; i += 8) {++ for (int l = 0; l < 4; ++l) {+ yl[l+0] = y[l+ 0];+ yl[l+4] = y[l+16];+ yh[l+0] = y[l+32];+ yh[l+4] = y[l+48];+ }++ device const half * dh = &x[i].d;+ device const uint8_t * q = x[i].qs + il;+ device const uint8_t * h = x[i].qh + in;+ device const int8_t * s = x[i].scales;++ for (int row = 0; row < 2; ++row) {++ const float d = dh[0];++ float2 acc = {0.f, 0.f};+ for (int l = 0; l < 4; ++l) {+ const uint8_t hl = h[l] >> iq;+ acc[0] += yl[l+0] * s[0] * ((int16_t)(q[l+ 0] & 0x0F) - (hl & 0x01 ? 0 : 16))+ + yl[l+4] * s[1] * ((int16_t)(q[l+16] & 0x0F) - (hl & 0x04 ? 0 : 16));+ acc[1] += yh[l+0] * s[2] * ((int16_t)(q[l+ 0] & 0xF0) - (hl & 0x10 ? 0 : 256))+ + yh[l+4] * s[3] * ((int16_t)(q[l+16] & 0xF0) - (hl & 0x40 ? 0 : 256));+ }+ sumf[row] += d * (acc[0] + 1.f/16.f * acc[1]);++ q += step;+ h += step;+ s += step;+ dh += step/2;++ }++ y += 8 * QK_K;+ }+#endif++ for (int row = 0; row < 2; ++row) {+ const float tot = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;+ }+ }+}++[[host_name("kernel_mul_mv_q5_K_f32")]]+kernel void kernel_mul_mv_q5_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q5_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}-constexpr constant static uint32_t iq3xs_grid[512] = {- 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14,- 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414,- 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24,- 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c,- 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c,- 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34,- 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c,- 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414,- 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,- 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404,- 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434,- 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c,- 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404,- 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414,- 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414,- 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404,- 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c,- 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,- 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404,- 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e,- 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14,- 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c,- 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424,- 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c,- 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c,- 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e,- 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,- 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e,- 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424,- 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e,- 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424,- 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404,- 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c,- 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e,- 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c,- 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,- 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c,- 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404,- 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04,- 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c,- 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414,- 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c,- 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c,- 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424,- 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,- 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c,- 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414,- 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c,- 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e,- 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04,- 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424,- 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14,- 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34,- 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,- 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434,- 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c,- 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424,- 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24,- 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24,- 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e,- 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c,- 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c,- 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,- 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,-};+void kernel_mul_mv_q6_K_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {-#define NGRID_IQ1S 512-constexpr constant static uint64_t iq1s_grid[NGRID_IQ1S] = {- 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,- 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,- 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,- 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,- 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,- 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,- 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,- 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,- 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,- 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,- 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,- 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,- 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,- 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,- 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,- 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,- 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,- 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,- 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,- 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,- 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,- 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,- 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,- 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,- 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,- 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,- 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,- 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,- 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,- 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,- 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,- 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,- 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,- 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,- 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,- 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,- 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,- 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,- 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,- 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,- 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,- 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,- 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,- 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,- 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,- 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,- 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,- 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,- 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,- 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,- 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,- 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,- 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,- 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,- 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,- 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,- 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,- 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,- 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,- 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,- 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,- 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,- 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,- 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,- 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,- 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,- 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,- 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,- 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,- 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,- 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,- 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,- 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,- 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,- 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,- 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,- 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,- 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,- 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,- 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,- 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,- 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,- 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,- 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,- 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,- 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,- 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,- 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,- 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,- 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,- 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,- 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,- 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,- 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,- 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,- 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,- 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,- 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,- 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,- 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,- 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,- 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,- 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,- 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,- 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,- 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,- 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,- 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,- 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,- 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,- 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,- 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,- 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,- 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,- 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,- 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,- 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,- 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,- 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,- 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,- 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,- 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,- 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,- 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,- 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,- 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,- 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,- 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,-};+ const uint8_t kmask1 = 0x03;+ const uint8_t kmask2 = 0x0C;+ const uint8_t kmask3 = 0x30;+ const uint8_t kmask4 = 0xC0;-constexpr constant static uint8_t ksigns_iq2xs[128] = {- 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,- 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,- 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,- 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,- 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,- 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,- 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,- 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,-};+ const int nb = ne00/QK_K;++ const int64_t r0 = tgpig.x;+ const int64_t r1 = tgpig.y;+ const int im = tgpig.z;++ const int row = 2 * r0 + sgitg;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_q6_K * x = (device const block_q6_K *) src0 + row * nb + offset0;+ device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float sumf = 0;++#if QK_K == 256+ const int tid = tiisg/2;+ const int ix = tiisg%2;+ const int ip = tid/8; // 0 or 1+ const int il = tid%8;+ const int n = 4;+ const int l0 = n*il;+ const int is = 8*ip + l0/16;++ const int y_offset = 128*ip + l0;+ const int q_offset_l = 64*ip + l0;+ const int q_offset_h = 32*ip + l0;++ for (int i = ix; i < nb; i += 2) {++ device const uint8_t * q1 = x[i].ql + q_offset_l;+ device const uint8_t * q2 = q1 + 32;+ device const uint8_t * qh = x[i].qh + q_offset_h;+ device const int8_t * sc = x[i].scales + is;++ device const float * y = yy + i * QK_K + y_offset;-constexpr constant static uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};+ const float dall = x[i].d;++ float4 sums = {0.f, 0.f, 0.f, 0.f};+ for (int l = 0; l < n; ++l) {+ sums[0] += y[l+ 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32);+ sums[1] += y[l+32] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32);+ sums[2] += y[l+64] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32);+ sums[3] += y[l+96] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32);+ }++ sumf += dall * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]);++ }++#else+ const int ix = tiisg/4;+ const int il = 4*(tiisg%4);++ for (int i = ix; i < nb; i += 8) {+ device const float * y = yy + i * QK_K + il;+ device const uint8_t * ql = x[i].ql + il;+ device const uint8_t * qh = x[i].qh + il;+ device const int8_t * s = x[i].scales;++ const float d = x[i].d;++ float4 sums = {0.f, 0.f, 0.f, 0.f};+ for (int l = 0; l < 4; ++l) {+ sums[0] += y[l+ 0] * ((int8_t)((ql[l+ 0] & 0xF) | ((qh[l] & kmask1) << 4)) - 32);+ sums[1] += y[l+16] * ((int8_t)((ql[l+16] & 0xF) | ((qh[l] & kmask2) << 2)) - 32);+ sums[2] += y[l+32] * ((int8_t)((ql[l+ 0] >> 4) | ((qh[l] & kmask3) >> 0)) - 32);+ sums[3] += y[l+48] * ((int8_t)((ql[l+16] >> 4) | ((qh[l] & kmask4) >> 2)) - 32);+ }+ sumf += d * (sums[0] * s[0] + sums[1] * s[1] + sums[2] * s[2] + sums[3] * s[3]);+ }++#endif++ const float tot = simd_sum(sumf);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + row] = tot;+ }+}++[[host_name("kernel_mul_mv_q6_K_f32")]]+kernel void kernel_mul_mv_q6_K_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_q6_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, tgpig, tiisg, sgitg);+}++// ======================= "True" 2-bitvoid kernel_mul_mv_iq2_xxs_f32_impl(device const void * src0,@@ -4346,7 +5583,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl(threadgroup_barrier(mem_flags::mem_threadgroup);}-#if QK_K == 256const int ix = tiisg;device const float * y4 = y + 32 * ix;@@ -4387,12 +5623,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl(y4 += 32 * 32;}-#else- (void) x;- (void) y;- (void) yl;- (void) nb32;-#endiffor (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);@@ -4482,7 +5712,6 @@ void kernel_mul_mv_iq2_xs_f32_impl(threadgroup_barrier(mem_flags::mem_threadgroup);}-#if QK_K == 256const int ix = tiisg;device const float * y4 = y + 32 * ix;@@ -4533,12 +5762,6 @@ void kernel_mul_mv_iq2_xs_f32_impl(y4 += 32 * 32;}-#else- (void) x;- (void) y;- (void) yl;- (void) nb32;-#endiffor (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);@@ -4628,7 +5851,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl(threadgroup_barrier(mem_flags::mem_threadgroup);}-#if QK_K == 256const int ix = tiisg;device const float * y4 = y + 32 * ix;@@ -4672,12 +5894,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl(y4 += 32 * 32;}-#else- (void) x;- (void) y;- (void) yl;- (void) nb32;-#endiffor (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);@@ -4759,7 +5975,7 @@ void kernel_mul_mv_iq3_s_f32_impl({int nval = 8;int pos = (32*sgitg + tiisg)*nval;- for (int i = 0; i < nval; ++i) values[pos + i] = iq3xs_grid[pos + i];+ for (int i = 0; i < nval; ++i) values[pos + i] = iq3s_grid[pos + i];threadgroup_barrier(mem_flags::mem_threadgroup);}@@ -4786,12 +6002,14 @@ void kernel_mul_mv_iq3_s_f32_impl(for (int row = 0; row < N_DST; row++) {const float db = dh[0];- const float d = db * (0.5f + ((sc[0] >> 4*(ib%2)) & 0xf));+ const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf));float2 sum = {0};for (int l = 0; l < 4; ++l) {- const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(values + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256)));- const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(values + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256)));+ const threadgroup uint32_t * table1 = qh[0] & kmask_iq2xs[2*l+0] ? values + 256 : values;+ const threadgroup uint32_t * table2 = qh[0] & kmask_iq2xs[2*l+1] ? values + 256 : values;+ const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(table1 + qs[2*l+0]);+ const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(table2 + qs[2*l+1]);for (int j = 0; j < 4; ++j) {sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]);sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]);@@ -4812,7 +6030,7 @@ void kernel_mul_mv_iq3_s_f32_impl(for (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);if (tiisg == 0) {- dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.5f;+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;}}}@@ -5011,56 +6229,54 @@ void kernel_mul_mv_iq1_s_f32_impl(device const block_iq1_s * x = (device const block_iq1_s *) src0 + ib_row + offset0;device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;- float yl[16];+ float yl[32];float sumf[N_DST]={0.f}, all_sum;const int nb32 = nb * (QK_K / 32);-#if QK_K == 256- const int ix = tiisg/2;- const int il = tiisg%2;+ const int ix = tiisg;- device const float * y4 = y + 32 * ix + 16 * il;+ device const float * y4 = y + 32 * ix;- for (int ib32 = ix; ib32 < nb32; ib32 += 16) {+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {- for (int i = 0; i < 16; ++i) {+ float sumy = 0;+ for (int i = 0; i < 32; ++i) {yl[i] = y4[i];+ sumy += yl[i];}const int ibl = ib32 / (QK_K / 32);const int ib = ib32 % (QK_K / 32);device const block_iq1_s * xr = x + ibl;- device const uint8_t * qs = xr->qs + 4 * ib + 2 * il;- device const uint8_t * sc = xr->scales + 2 * ib + il;- device const half * dh = &xr->d;+ device const uint8_t * qs = xr->qs + 4 * ib;+ device const uint16_t * qh = xr->qh + ib;+ device const half * dh = &xr->d;for (int row = 0; row < N_DST; row++) {- constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5)));- constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1)));+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700)));+ constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700)));+ constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[0] >> 1) & 0x700)));- float2 sum = {0};- for (int j = 0; j < 8; ++j) {- sum[0] += yl[j+ 0] * grid1[j];- sum[1] += yl[j+ 8] * grid2[j];+ float sum = 0;+ for (int j = 0; j < 4; ++j) {+ sum += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4)+ + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4)+ + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4)+ + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4);}- sumf[row] += (float)dh[0] * (sum[0] * (2*(sc[0] & 7) + 1) + sum[1] * (2*((sc[0] >> 4) & 7) + 1));+ sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1);dh += nb*sizeof(block_iq1_s)/2;qs += nb*sizeof(block_iq1_s);- sc += nb*sizeof(block_iq1_s);+ qh += nb*sizeof(block_iq1_s)/2;}- y4 += 16 * 32;+ y4 += 32 * 32;}-#else- (void) x;- (void) y;- (void) yl;- (void) nb32;-#endiffor (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);@@ -5070,10 +6286,6 @@ void kernel_mul_mv_iq1_s_f32_impl(}}-constexpr constant static float kvalues_iq4nl_f[16] = {- -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f-};-void kernel_mul_mv_iq4_nl_f32_impl(device const void * src0,device const float * src1,@@ -5167,6 +6379,7 @@ void kernel_mul_mv_iq4_nl_f32_impl(}}+#if QK_K != 64void kernel_mul_mv_iq4_xs_f32_impl(device const void * src0,device const float * src1,@@ -5260,6 +6473,7 @@ void kernel_mul_mv_iq4_xs_f32_impl(}}}+#endif[[host_name("kernel_mul_mv_iq1_s_f32")]]kernel void kernel_mul_mv_iq1_s_f32(@@ -5344,7 +6558,11 @@ kernel void kernel_mul_mv_iq4_xs_f32(uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {+#if QK_K == 64+ kernel_mul_mv_iq4_nl_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);+#elsekernel_mul_mv_iq4_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);+#endif}//============================= templates and their specializations =============================@@ -5703,15 +6921,15 @@ void dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 &device const uint8_t * qs = xb->qs + 8*ib32;device const uint8_t * signs = xb->signs + 4*ib32 + 2*il;const uint8_t qh = xb->qh[ib32] >> 4*il;- const float dl = d * (0.5f + ((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * 0.5f;- constant uint8_t * grid1 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+0] | ((qh << 8) & 256)));- constant uint8_t * grid2 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+1] | ((qh << 7) & 256)));+ const float dl = d * (1 + 2*((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf));+ constant uint8_t * grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+0] | ((qh << 8) & 256)));+ constant uint8_t * grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+1] | ((qh << 7) & 256)));for (int i = 0; i < 4; ++i) {reg[0][i] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i+0]);reg[1][i] = dl * grid2[i] * select(1, -1, signs[0] & kmask_iq2xs[i+4]);}- grid1 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+2] | ((qh << 6) & 256)));- grid2 = (constant uint8_t *)(iq3xs_grid + (qs[4*il+3] | ((qh << 5) & 256)));+ grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+2] | ((qh << 6) & 256)));+ grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+3] | ((qh << 5) & 256)));for (int i = 0; i < 4; ++i) {reg[2][i] = dl * grid1[i] * select(1, -1, signs[1] & kmask_iq2xs[i+0]);reg[3][i] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i+4]);@@ -5740,16 +6958,21 @@ void dequantize_iq2_s(device const block_iq2_s * xb, short il, thread type4x4 &template <typename type4x4>void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) {// il is 0...15 for QK_K = 256 => index of block of 32 is il/2+ const int ib32 = il/2;+ il = il%2;const float d = xb->d;- device const uint8_t * qs = xb->qs + 2*il;- device const uint8_t * sc = xb->scales + il;- const float dl1 = d * (2*(sc[0] & 7) + 1);- const float dl2 = d * (2*((sc[0] >> 4) & 7) + 1);- constant int8_t * grid1 = (constant int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5)));- constant int8_t * grid2 = (constant int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1)));- for (int i = 0; i < 8; ++i) {- reg[i/4+0][i%4] = dl1 * grid1[i];- reg[i/4+2][i%4] = dl2 * grid2[i];+ device const uint8_t * qs = xb->qs + 4*ib32 + 2*il;+ device const uint16_t * qh = xb->qh;+ const float dl = d * (2*((qh[ib32] >> 12) & 7) + 1);+ const float ml = dl * (qh[ib32] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA);+ const uint16_t h = qh[ib32] >> 6*il;+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((h << 8) & 0x700)));+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((h << 5) & 0x700)));+ for (int i = 0; i < 4; ++i) {+ reg[0][i] = dl * (grid1[i] & 0xf) + ml;+ reg[1][i] = dl * (grid1[i] >> 4) + ml;+ reg[2][i] = dl * (grid2[i] & 0xf) + ml;+ reg[3][i] = dl * (grid2[i] >> 4) + ml;}}@@ -5770,6 +6993,9 @@ void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4template <typename type4x4>void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) {+#if QK_K == 64+ dequantize_iq4_nl(xb, il, reg);+#else// il is 0...15 for QK_K = 256 => index of block of 32 is il/2const int ib32 = il/2;il = il%2;@@ -5786,6 +7012,7 @@ void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4reg[i][2] = d * kvalues_iq4nl_f[q8[2]];reg[i][3] = d * kvalues_iq4nl_f[q8[3]];}+#endif}template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>@@ -6056,7 +7283,7 @@ template<typename block_q, short nl, void (*dequantize_func)(device const block_void kernel_mul_mm_id_impl(device const uchar * src0,device const uchar * src1,- thread short * src1ids,+ threadgroup short * src1ids,device float * dst,constant int64_t & ne00,constant int64_t & ne02,@@ -6259,9 +7486,9 @@ kernel void kernel_mul_mm_id(tgpig.z = tgpig.z%(ne12*ne13);// row indices of src1 for expert id- int64_t _ne1 = 0;- short src1ids[512];+ threadgroup short * src1ids = (threadgroup short *)(shared_memory + 8192);+ int64_t _ne1 = 0;for (int64_t i1 = 0; i1 < ne1; i1++) {if (((device int32_t *) (ids + i1*nbi1))[idx] == id) {src1ids[_ne1++] = i1;@@ -6334,7 +7561,11 @@ template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_t kernel_get_rtemplate [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_t kernel_get_rows<block_iq2_s, QK_NL, dequantize_iq2_s>;template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows<block_iq1_s, QK_NL, dequantize_iq1_s>;template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows<block_iq4_nl, 2, dequantize_iq4_nl>;+#if QK_K == 64+template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_t kernel_get_rows<block_iq4_xs, 2, dequantize_iq4_xs>;+#elsetemplate [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_t kernel_get_rows<block_iq4_xs, QK_NL, dequantize_iq4_xs>;+#endif//// matrix-matrix multiplication@@ -6378,7 +7609,11 @@ template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mat_mm_t kernel_mul_mtemplate [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq2_s, QK_NL, dequantize_iq2_s>;template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq1_s, QK_NL, dequantize_iq1_s>;template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq4_nl, 2, dequantize_iq4_nl>;+#if QK_K == 64+template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq4_nl, 2, dequantize_iq4_xs>;+#elsetemplate [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq4_xs, QK_NL, dequantize_iq4_xs>;+#endif//// indirect matrix-matrix multiplication@@ -6434,7 +7669,11 @@ template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mat_mm_id_t kerneltemplate [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_s, QK_NL, dequantize_iq2_s>;template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq1_s, QK_NL, dequantize_iq1_s>;template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq4_nl, 2, dequantize_iq4_nl>;+#if QK_K == 64+template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq4_xs, 2, dequantize_iq4_xs>;+#elsetemplate [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq4_xs, QK_NL, dequantize_iq4_xs>;+#endif//// matrix-vector multiplication@@ -7707,7 +8946,11 @@ kernel void kernel_mul_mv_id_iq4_xs_f32(const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];+#if QK_K == 64+ kernel_mul_mv_iq4_nl_f32_impl(+#elsekernel_mul_mv_iq4_xs_f32_impl(+#endifsrc0[id],(device const float *) (src1 + bid*nb11),dst + bid*ne0,diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex daf94e69..d8ef1ec6 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -1,5 +1,9 @@// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi++#define GGML_COMMON_IMPL_C+#include "ggml-common.h"+#include "ggml-quants.h"#include "ggml-impl.h"@@ -1949,8 +1953,7 @@ static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restri}}-size_t quantize_q2_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_q2_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {size_t row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row);if (!quant_weights) {quantize_row_q2_K_reference(src, dst, nrow*n_per_row);@@ -2268,8 +2271,7 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri#endif}-size_t quantize_q3_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_q3_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {size_t row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row);if (!quant_weights) {quantize_row_q3_K_reference(src, dst, nrow*n_per_row);@@ -2535,8 +2537,7 @@ static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restri#endif}-size_t quantize_q4_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_q4_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {size_t row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row);if (!quant_weights) {quantize_row_q4_K_reference(src, dst, nrow*n_per_row);@@ -2858,8 +2859,7 @@ static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restri#endif}-size_t quantize_q5_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_q5_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {size_t row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row);if (!quant_weights) {quantize_row_q5_K_reference(src, dst, nrow*n_per_row);@@ -3112,8 +3112,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri#endif}-size_t quantize_q6_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_q6_K(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {size_t row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row);if (!quant_weights) {quantize_row_q6_K_reference(src, dst, nrow*n_per_row);@@ -3157,9 +3156,10 @@ static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restri}}-size_t quantize_q4_0(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+size_t quantize_q4_0(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {if (!quant_weights) {- return ggml_quantize_q4_0(src, dst, nrow*n_per_row, n_per_row, hist);+ quantize_row_q4_0_reference(src, dst, nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q4_0, n_per_row);}size_t row_size = ggml_row_size(GGML_TYPE_Q4_0, n_per_row);char * qrow = (char *)dst;@@ -3201,9 +3201,10 @@ static void quantize_row_q4_1_impl(const float * restrict x, block_q4_1 * restri}}-size_t quantize_q4_1(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+size_t quantize_q4_1(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {if (!quant_weights) {- return ggml_quantize_q4_1(src, dst, nrow*n_per_row, n_per_row, hist);+ quantize_row_q4_1_reference(src, dst, nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q4_1, n_per_row);}size_t row_size = ggml_row_size(GGML_TYPE_Q4_1, n_per_row);char * qrow = (char *)dst;@@ -3254,9 +3255,10 @@ static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restri}}-size_t quantize_q5_0(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+size_t quantize_q5_0(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {if (!quant_weights) {- return ggml_quantize_q5_0(src, dst, nrow*n_per_row, n_per_row, hist);+ quantize_row_q5_0_reference(src, dst, nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q5_0, n_per_row);}size_t row_size = ggml_row_size(GGML_TYPE_Q5_0, n_per_row);char * qrow = (char *)dst;@@ -3306,9 +3308,10 @@ static void quantize_row_q5_1_impl(const float * restrict x, block_q5_1 * restri}}-size_t quantize_q5_1(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+size_t quantize_q5_1(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {if (!quant_weights) {- return ggml_quantize_q5_1(src, dst, nrow*n_per_row, n_per_row, hist);+ quantize_row_q5_1_reference(src, dst, nrow*n_per_row);+ return nrow * ggml_row_size(GGML_TYPE_Q5_1, n_per_row);}size_t row_size = ggml_row_size(GGML_TYPE_Q5_1, n_per_row);char * qrow = (char *)dst;@@ -3320,712 +3323,14 @@ size_t quantize_q5_1(const float * src, void * dst, int nrow, int n_per_row, intreturn nrow * row_size;}-// ====================== "True" 2-bit (de)-quantization--static const uint64_t iq2xxs_grid[256] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,- 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,- 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,- 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,- 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,- 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,- 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,- 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,- 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,- 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,- 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,- 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,- 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,- 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,- 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,- 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,- 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,- 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,- 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,- 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,- 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,- 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,- 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,- 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,- 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,- 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,- 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,- 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,- 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,- 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,- 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,- 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,- 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,- 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,- 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,- 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,- 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,- 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,- 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,- 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,- 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,- 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,- 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,- 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,- 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,- 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,- 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,- 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,- 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,- 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,- 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,- 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,- 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,- 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,- 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,- 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,- 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,- 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,- 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,- 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,- 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,- 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,- 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,-};--static const uint64_t iq2xs_grid[512] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,- 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,- 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,- 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,- 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,- 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,- 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,- 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,- 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,- 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,- 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,- 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,- 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,- 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,- 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,- 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,- 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,- 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,- 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,- 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,- 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,- 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,- 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,- 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,- 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,- 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,- 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,- 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,- 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,- 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,- 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,- 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,- 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,- 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,- 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,- 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,- 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,- 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,- 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,- 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,- 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,- 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,- 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,- 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,- 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,- 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,- 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,- 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,- 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,- 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,- 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,- 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,- 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,- 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,- 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,- 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,- 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,- 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,- 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,- 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,- 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,- 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,- 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,- 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,- 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,- 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,- 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,- 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,- 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,- 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,- 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,- 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,- 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,- 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,- 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,- 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,- 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,- 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,- 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,- 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,- 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,- 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,- 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,- 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,- 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,- 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,- 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,- 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,- 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,- 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,- 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,- 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,- 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,- 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,- 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,- 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,- 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,- 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,- 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,- 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,- 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,- 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,- 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,- 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,- 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,- 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,- 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,- 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,- 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,- 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,- 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,- 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,- 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,- 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,- 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,- 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,- 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,- 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,- 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,- 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,- 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,- 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,-};--static const uint64_t iq2s_grid[1024] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,- 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,- 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,- 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,- 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,- 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,- 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,- 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,- 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,- 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,- 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,- 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,- 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,- 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,- 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,- 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,- 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,- 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,- 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,- 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,- 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,- 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,- 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,- 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,- 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,- 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,- 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,- 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,- 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,- 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,- 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,- 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,- 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,- 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,- 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,- 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,- 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,- 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,- 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,- 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,- 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,- 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,- 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,- 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,- 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,- 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,- 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,- 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,- 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,- 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,- 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,- 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,- 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,- 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,- 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,- 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,- 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,- 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,- 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,- 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,- 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,- 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,- 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,- 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,- 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,- 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,- 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,- 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,- 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,- 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,- 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,- 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,- 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,- 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,- 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,- 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,- 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,- 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,- 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,- 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,- 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,- 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,- 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,- 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,- 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,- 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,- 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,- 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,- 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,- 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,- 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,- 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,- 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,- 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,- 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,- 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,- 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,- 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,- 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,- 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,- 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,- 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,- 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,- 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,- 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,- 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,- 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,- 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,- 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,- 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,- 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,- 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,- 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,- 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,- 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,- 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,- 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,- 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,- 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,- 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,- 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,- 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,- 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,- 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,- 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,- 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,- 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,- 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,- 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,- 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,- 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,- 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,- 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,- 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,- 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,- 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,- 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,- 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,- 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,- 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,- 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,- 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,- 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,- 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,- 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,- 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,- 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,- 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,- 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,- 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,- 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,- 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,- 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,- 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,- 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,- 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,- 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,- 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,- 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,- 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,- 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,- 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,- 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,- 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,- 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,- 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,- 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,- 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,- 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,- 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,- 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,- 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,- 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,- 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,- 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,- 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,- 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,- 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,- 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,- 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,- 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,- 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,- 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,- 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,- 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,- 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,- 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,- 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,- 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,- 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,- 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,- 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,- 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,- 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,- 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,- 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,- 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,- 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,- 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,- 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,- 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,- 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,- 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,- 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,- 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,- 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,- 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,- 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,- 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,- 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,- 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,- 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,- 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,- 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,- 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,- 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,- 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,- 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,- 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,- 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,- 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,- 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,- 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,- 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,- 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,- 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,- 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,- 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,- 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,- 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,- 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,- 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,- 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,- 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,- 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,- 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,- 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,- 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,- 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,- 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,- 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,- 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,- 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,- 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,- 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,- 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,- 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,- 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,- 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,- 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,- 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,- 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,- 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,- 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,-};--static const uint32_t iq3xxs_grid[256] = {- 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,- 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,- 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,- 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,- 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,- 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,- 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,- 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,- 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,- 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,- 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,- 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,- 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,- 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,- 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,- 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,- 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,- 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,- 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,- 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,- 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,- 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,- 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,- 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,- 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,- 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,- 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,- 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,- 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,- 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,- 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,- 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,-};--static const uint32_t iq3xs_grid[512] = {- 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14,- 0x04040c24, 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414,- 0x0404242c, 0x0404243e, 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24,- 0x04043e3e, 0x040c0404, 0x040c040c, 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c,- 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c,- 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, 0x04140c04, 0x04140c1c, 0x04140c34,- 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, 0x0414243e, 0x04142c0c,- 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, 0x041c1414,- 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,- 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404,- 0x0424241c, 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434,- 0x042c1c1c, 0x042c240c, 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c,- 0x04340c1c, 0x04341c0c, 0x04342c14, 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404,- 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414,- 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, 0x0c040c3e, 0x0c041404, 0x0c041414,- 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, 0x0c043e14, 0x0c0c0404,- 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, 0x0c0c1c1c,- 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,- 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404,- 0x0c143e14, 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e,- 0x0c1c1c04, 0x0c1c1c24, 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14,- 0x0c240c24, 0x0c241c0c, 0x0c241c1c, 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c,- 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424,- 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c,- 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, 0x1404041c, 0x1404042c,- 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, 0x1404143e,- 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,- 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e,- 0x140c1414, 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424,- 0x1414043e, 0x1414140c, 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e,- 0x14143e0c, 0x14143e24, 0x141c0404, 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424,- 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404,- 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, 0x14243e2c, 0x142c0424, 0x142c0c0c,- 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, 0x14340414, 0x1434043e,- 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, 0x143e241c,- 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,- 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c,- 0x1c0c040c, 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404,- 0x1c0c3e14, 0x1c0c3e34, 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04,- 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c,- 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414,- 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c,- 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, 0x1c3e040c, 0x1c3e041c,- 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, 0x24041424,- 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,- 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c,- 0x2414041c, 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414,- 0x24143e04, 0x241c0424, 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c,- 0x24240404, 0x24240414, 0x24241424, 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e,- 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04,- 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, 0x2c041414, 0x2c042404, 0x2c042424,- 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, 0x2c0c042c, 0x2c0c0c14,- 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, 0x2c141c34,- 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,- 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434,- 0x2c2c2c0c, 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c,- 0x34040c2c, 0x34041c0c, 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424,- 0x34140c14, 0x34141c24, 0x34142414, 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24,- 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24,- 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, 0x3e040404, 0x3e040424, 0x3e04043e,- 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, 0x3e0c0414, 0x3e0c0c0c,- 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, 0x3e14140c,- 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,- 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,-};--#define NGRID_IQ2XXS 512-static const uint64_t iq1s_grid[NGRID_IQ2XXS] = {- 0xffffffffffff0101, 0xffffffffff01ff00, 0xffffffffff010100, 0xffffffff00000000,- 0xffffffff01ff00ff, 0xffffffff01ff0001, 0xffffffff0101ffff, 0xffffffff0101ff01,- 0xffffff00ff000000, 0xffffff000000ff00, 0xffffff00000000ff, 0xffffff0000000100,- 0xffffff0000010000, 0xffffff0001000000, 0xffffff01ffff00ff, 0xffffff01ff01ff00,- 0xffffff01ff010100, 0xffffff0100000001, 0xffffff0101ffff00, 0xffffff0101ff0101,- 0xffffff0101010100, 0xffff00ffff00ff01, 0xffff00ffff0000ff, 0xffff00ff00ff0100,- 0xffff00ff0100ff00, 0xffff00ff010001ff, 0xffff0000ff0101ff, 0xffff000000ffff00,- 0xffff000000000000, 0xffff00000001ff01, 0xffff000001000101, 0xffff0000010100ff,- 0xffff0001ffff0100, 0xffff00010000ff00, 0xffff000100010101, 0xffff000101000000,- 0xffff01ffffff0000, 0xffff01ffff01ffff, 0xffff01ffff010100, 0xffff01ff00000000,- 0xffff01ff01ffffff, 0xffff01ff01ff0001, 0xffff01ff0101ffff, 0xffff01ff01010001,- 0xffff0100ffffff01, 0xffff01000000ffff, 0xffff010000000100, 0xffff010001ff01ff,- 0xffff010001000000, 0xffff0101ff000000, 0xffff0101000101ff, 0xffff010101ffff01,- 0xffff01010101ff00, 0xff00ffffff000000, 0xff00ffff00ffff00, 0xff00ffff00000001,- 0xff00ffff000001ff, 0xff00ffff01010000, 0xff00ff00ffff0000, 0xff00ff00ff00ff00,- 0xff00ff00ff0000ff, 0xff00ff00ff000100, 0xff00ff00ff010001, 0xff00ff0000ff0001,- 0xff00ff000000ffff, 0xff00ff0000000000, 0xff00ff000001ff00, 0xff00ff0000010100,- 0xff00ff0001ff0000, 0xff00ff000100ff00, 0xff00ff0001000100, 0xff00ff01ff000000,- 0xff00ff0100ff0000, 0xff00ff01000001ff, 0xff00ff0101010001, 0xff0000ff00000000,- 0xff0000ff0001ff00, 0xff0000ff00010100, 0xff000000ffff0101, 0xff000000ff000000,- 0xff000000ff01ff00, 0xff00000000ff0000, 0xff0000000000ff00, 0xff000000000000ff,- 0xff00000000000000, 0xff00000000000001, 0xff00000000000100, 0xff0000000001ffff,- 0xff00000000010000, 0xff00000001000000, 0xff00000001010100, 0xff000001ff00ff01,- 0xff000001ff0100ff, 0xff00000100000000, 0xff0000010001ff00, 0xff00000101ff0100,- 0xff0000010100ff00, 0xff0001ff00ff00ff, 0xff0001ff00000101, 0xff0001ff000100ff,- 0xff0001ff01000000, 0xff000100ff0001ff, 0xff0001000000ff01, 0xff00010000000000,- 0xff00010000010001, 0xff00010000010100, 0xff00010001ffff00, 0xff00010001ff0101,- 0xff00010001010000, 0xff000101ffffffff, 0xff000101ff000101, 0xff00010101ff00ff,- 0xff00010101000001, 0xff000101010100ff, 0xff01ffffff000101, 0xff01ffffff01ffff,- 0xff01ffffff01ff01, 0xff01ffffff0101ff, 0xff01ffff00000000, 0xff01ffff01ff0001,- 0xff01ffff0101ff01, 0xff01ff00ff000000, 0xff01ff0000ff0100, 0xff01ff000000ff01,- 0xff01ff0000010000, 0xff01ff00010000ff, 0xff01ff01ff01ff00, 0xff01ff0100000101,- 0xff0100ffffff0000, 0xff0100ffff010000, 0xff0100ff01ff00ff, 0xff0100ff01000100,- 0xff0100ff010100ff, 0xff010000ffffff01, 0xff01000000000000, 0xff0100000101ff00,- 0xff010001ffff00ff, 0xff010001ff000100, 0xff01000100ffff00, 0xff01000100010001,- 0xff01000101ff0001, 0xff010001010001ff, 0xff0101ffffffffff, 0xff0101ffff01ffff,- 0xff0101ffff010101, 0xff0101ff0000ff00, 0xff0101ff01010001, 0xff010100ff000000,- 0xff010100ff01ff01, 0xff01010000ff0001, 0xff01010000000100, 0xff01010001000000,- 0xff0101010100ffff, 0x00ffffff0000ff01, 0x00ffffff000000ff, 0x00ffffff00000100,- 0x00ffffff00010000, 0x00ffff00ffff0001, 0x00ffff00ff0000ff, 0x00ffff00ff000100,- 0x00ffff0000000000, 0x00ffff0001000100, 0x00ffff0001010001, 0x00ffff01ff00ff01,- 0x00ffff0100ff0100, 0x00ffff010000ff00, 0x00ffff01000100ff, 0x00ffff0101ff00ff,- 0x00ffff010101ff00, 0x00ff00ffffffffff, 0x00ff00ffffff01ff, 0x00ff00ffff000101,- 0x00ff00ff00000000, 0x00ff00ff000101ff, 0x00ff00ff01010101, 0x00ff0000ff000000,- 0x00ff0000ff01ffff, 0x00ff000000ff0000, 0x00ff00000000ff00, 0x00ff0000000000ff,- 0x00ff000000000000, 0x00ff000000000001, 0x00ff000000000100, 0x00ff000000010000,- 0x00ff000001ffff01, 0x00ff000001000000, 0x00ff0001ff000101, 0x00ff000100ffffff,- 0x00ff000100000000, 0x00ff0001010001ff, 0x00ff01ffff000000, 0x00ff01ff0001ff00,- 0x00ff01ff01ff0100, 0x00ff0100ff01ff01, 0x00ff010000ff00ff, 0x00ff010000ff0101,- 0x00ff010000000000, 0x00ff010000010101, 0x00ff01000100ff00, 0x00ff010001010000,- 0x00ff0101ffffff00, 0x00ff01010000ff01, 0x00ff010100000100, 0x00ff010101ff0000,- 0x0000ffffffff0100, 0x0000ffffff00ff00, 0x0000ffffff0000ff, 0x0000ffffff010000,- 0x0000ffff00000000, 0x0000ffff00010101, 0x0000ffff01ffff01, 0x0000ffff01000100,- 0x0000ff00ff000000, 0x0000ff00ff01ff00, 0x0000ff00ff0101ff, 0x0000ff0000ff0000,- 0x0000ff000000ff00, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,- 0x0000ff0000000100, 0x0000ff0000010000, 0x0000ff0001ffffff, 0x0000ff0001ff01ff,- 0x0000ff0001000000, 0x0000ff000101ffff, 0x0000ff01ffff0101, 0x0000ff01ff010000,- 0x0000ff0100000000, 0x0000ff0101000101, 0x000000ffffff0001, 0x000000ffff000000,- 0x000000ff00ff0000, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,- 0x000000ff00000001, 0x000000ff00000100, 0x000000ff00010000, 0x000000ff01000000,- 0x000000ff0101ff00, 0x00000000ffff0000, 0x00000000ff00ff00, 0x00000000ff0000ff,- 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff010000,- 0x0000000000ffff00, 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001,- 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,- 0x00000000000000ff, 0x0000000000000001, 0x00000000000001ff, 0x0000000000000100,- 0x0000000000000101, 0x000000000001ff00, 0x00000000000100ff, 0x0000000000010000,- 0x0000000000010001, 0x0000000000010100, 0x0000000001ff0000, 0x000000000100ff00,- 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, 0x0000000001000100,- 0x0000000001010000, 0x00000001ffff01ff, 0x00000001ff000000, 0x0000000100ff0000,- 0x000000010000ff00, 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001,- 0x0000000100000100, 0x0000000100010000, 0x0000000101000000, 0x000001ffff00ff00,- 0x000001ffff010001, 0x000001ffff0101ff, 0x000001ff00ffff01, 0x000001ff0000ffff,- 0x000001ff00000000, 0x000001ff010000ff, 0x000001ff01010100, 0x00000100ffff0100,- 0x00000100ff000000, 0x0000010000ff0000, 0x000001000000ff00, 0x00000100000000ff,- 0x0000010000000000, 0x0000010000000001, 0x0000010000000100, 0x0000010000010000,- 0x0000010001000000, 0x000001000101ff01, 0x00000101ffff0001, 0x00000101ff01ffff,- 0x0000010100000000, 0x0000010101010100, 0x0001ffffff000000, 0x0001ffff00ffffff,- 0x0001ffff00000100, 0x0001ffff0001ff00, 0x0001ffff01000000, 0x0001ff00ffffff00,- 0x0001ff00ffff01ff, 0x0001ff00ff010000, 0x0001ff0000000000, 0x0001ff0000010001,- 0x0001ff0001ff0000, 0x0001ff0001010100, 0x0001ff01ff0000ff, 0x0001ff01ff000001,- 0x0001ff0100ffffff, 0x0001ff010001ffff, 0x0001ff01000101ff, 0x0001ff010100ff01,- 0x000100ffff00ffff, 0x000100ffff00ff01, 0x000100ffff000100, 0x000100ff00000000,- 0x000100ff000101ff, 0x000100ff01ff0101, 0x000100ff0100ffff, 0x000100ff01010101,- 0x00010000ff000000, 0x00010000ff010100, 0x0001000000ff0000, 0x000100000000ff00,- 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, 0x0001000000000100,- 0x0001000000010000, 0x0001000001ffff01, 0x0001000001000000, 0x0001000100ff0101,- 0x0001000100000000, 0x00010001010100ff, 0x000101ffffff01ff, 0x000101ffffff0101,- 0x000101ff00010000, 0x000101ff01ff0000, 0x000101ff0100ff01, 0x00010100ffff0000,- 0x0001010000000000, 0x000101000001ffff, 0x0001010000010101, 0x00010100010001ff,- 0x00010101ff00ff00, 0x00010101ff010001, 0x0001010100ffffff, 0x0001010100ff01ff,- 0x00010101000101ff, 0x0001010101ff0000, 0x000101010100ff01, 0x0001010101000101,- 0x01ffffffffff0101, 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff,- 0x01ffffffff010101, 0x01ffffff00000000, 0x01ffffff01ff01ff, 0x01ffffff01000101,- 0x01ffffff0101ff01, 0x01ffffff010100ff, 0x01ffff000000ff00, 0x01ffff0000000001,- 0x01ffff00000001ff, 0x01ffff0000010000, 0x01ffff0001ff0000, 0x01ffff01ffffffff,- 0x01ffff01ffff01ff, 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff0101ff,- 0x01ffff010100ffff, 0x01ff00ffffff0000, 0x01ff00ffff010000, 0x01ff00ff00ffff01,- 0x01ff0000ff0000ff, 0x01ff000000000000, 0x01ff00000001ff01, 0x01ff000001ffffff,- 0x01ff000001010100, 0x01ff0001ffffff01, 0x01ff0001ff010001, 0x01ff000101ff0100,- 0x01ff000101000001, 0x01ff0001010100ff, 0x01ff01ffff00ffff, 0x01ff01ff00010001,- 0x01ff01ff01000000, 0x01ff01ff010101ff, 0x01ff0100ff000001, 0x01ff010000ffff00,- 0x01ff010000000100, 0x01ff010001ff01ff, 0x01ff01000101ffff, 0x01ff0101ffff00ff,- 0x01ff0101ffff0101, 0x01ff0101ff0101ff, 0x01ff010100010000, 0x0100ffff00ff00ff,- 0x0100ffff00ff0001, 0x0100ffff00000100, 0x0100ffff0100ff00, 0x0100ff00ffff0000,- 0x0100ff00ff00ffff, 0x0100ff00ff00ff01, 0x0100ff00ff000100, 0x0100ff00ff010000,- 0x0100ff0000000000, 0x0100ff00000100ff, 0x0100ff0001ff0101, 0x0100ff0001010101,- 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100000100, 0x0100ff0100010001,- 0x0100ff0101000000, 0x010000ffff00ff00, 0x010000ff0000ffff, 0x010000ff00000000,- 0x010000ff010001ff, 0x010000ff01010001, 0x01000000ffffff00, 0x01000000ffff0101,- 0x01000000ff000000, 0x01000000ff0100ff, 0x01000000ff010101, 0x0100000000ff0000,- 0x010000000000ff00, 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001,- 0x0100000000000100, 0x0100000000010000, 0x0100000001000000, 0x0100000100000000,- 0x01000001000101ff, 0x0100000101ffff01, 0x010001ffff000101, 0x010001ff00ff0100,- 0x010001ff0000ff00, 0x010001ff000100ff, 0x010001ff01ffffff, 0x01000100ffff0000,- 0x01000100ff0001ff, 0x0100010000000000, 0x010001000001ff00, 0x0100010001ff0000,- 0x01000100010000ff, 0x0100010001000101, 0x01000101ff00ff01, 0x0100010100ff0100,- 0x010001010000ffff, 0x0100010101010001, 0x0101ffffffff0101, 0x0101ffffff0001ff,- 0x0101ffffff01ffff, 0x0101ffffff010101, 0x0101ffff00000000, 0x0101ffff0101ffff,- 0x0101ffff010101ff, 0x0101ff00ff000000, 0x0101ff0000ff0100, 0x0101ff000000ff00,- 0x0101ff0000010000, 0x0101ff00010000ff, 0x0101ff0001000001, 0x0101ff01ff010101,- 0x0101ff0100000000, 0x0101ff010101ff00, 0x010100ffffff0000, 0x010100ffff010000,- 0x010100ff00ff01ff, 0x010100ff000000ff, 0x010100ff00000101, 0x010100ff01ffff00,- 0x01010000ffffff01, 0x01010000ff000100, 0x01010000ff01ff01, 0x0101000000000000,- 0x01010000000100ff, 0x010100000101ff01, 0x01010001ffff0000, 0x01010001ff00ffff,- 0x01010001ff010000, 0x0101000101ffffff, 0x0101000101ff01ff, 0x0101000101010101,- 0x010101ffff01ffff, 0x010101ff00000000, 0x010101ff0001ff01, 0x010101ff0101ffff,- 0x010101ff010101ff, 0x01010100ffffffff, 0x01010100ff000001, 0x010101000000ff00,- 0x0101010001010000, 0x0101010100ff0001, 0x010101010001ff01, 0x010101010101ffff,--};--static const uint8_t ksigns_iq2xs[128] = {- 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,- 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,- 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,- 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,- 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,- 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,- 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,- 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,-};+size_t quantize_q8_0(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {+ (void)quant_weights; // not used+ const size_t row_size = ggml_row_size(GGML_TYPE_Q8_0, n_per_row);+ quantize_row_q8_0_reference(src, dst, nrow*n_per_row);+ return nrow * row_size;+}-static const uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};+// ====================== "True" 2-bit (de)-quantizationvoid dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k) {assert(k % QK_K == 0);@@ -4158,11 +3463,11 @@ void dequantize_row_iq3_s(const block_iq3_s * restrict x, float * restrict y, inconst uint8_t * signs = x[i].signs;for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {- const float db1 = d * (0.5f + (x[i].scales[ib32/2] & 0xf)) * 0.5f;- const float db2 = d * (0.5f + (x[i].scales[ib32/2] >> 4)) * 0.5f;+ const float db1 = d * (1 + 2*(x[i].scales[ib32/2] & 0xf));+ const float db2 = d * (1 + 2*(x[i].scales[ib32/2] >> 4));for (int l = 0; l < 4; ++l) {- const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256)));- const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256)));+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256)));for (int j = 0; j < 4; ++j) {y[j+0] = db1 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);y[j+4] = db1 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);@@ -4172,8 +3477,8 @@ void dequantize_row_iq3_s(const block_iq3_s * restrict x, float * restrict y, inqs += 8;signs += 4;for (int l = 0; l < 4; ++l) {- const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256)));- const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256)));+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256)));for (int j = 0; j < 4; ++j) {y[j+0] = db2 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f);y[j+4] = db2 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f);@@ -4193,39 +3498,23 @@ void dequantize_row_iq1_s(const block_iq1_s * restrict x, float * restrict y, inassert(k % QK_K == 0);const int nb = k / QK_K;- float db[4];- uint16_t idx[4];- //const int8_t * grid[4];-for (int i = 0; i < nb; i++) {const float d = GGML_FP16_TO_FP32(x[i].d);- const uint8_t * sc = x[i].scales;- const uint8_t * qs = x[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;- for (int i8 = 0; i8 < QK_K/8; i8 += 4) {- idx[0] = qs[0] | ((sc[0] & 0x08) << 5);- idx[1] = qs[1] | ((sc[0] & 0x80) << 1);- idx[2] = qs[2] | ((sc[1] & 0x08) << 5);- idx[3] = qs[3] | ((sc[1] & 0x80) << 1);- //grid[0] = (const int8_t *)(iq1s_grid + (qs[0] | ((sc[0] & 0x08) << 5)));- //grid[1] = (const int8_t *)(iq1s_grid + (qs[1] | ((sc[0] & 0x80) << 1)));- //grid[2] = (const int8_t *)(iq1s_grid + (qs[2] | ((sc[1] & 0x08) << 5)));- //grid[3] = (const int8_t *)(iq1s_grid + (qs[3] | ((sc[1] & 0x80) << 1)));- db[0] = d * (2*(sc[0] & 7) + 1);- db[1] = d * (2*((sc[0] >> 4) & 7) + 1);- db[2] = d * (2*(sc[1] & 7) + 1);- db[3] = d * (2*((sc[1] >> 4) & 7) + 1);+ for (int ib = 0; ib < QK_K/32; ++ib) {+ const float dl = d * (2*((qh[ib] >> 12) & 7) + 1);+ const float delta = qh[ib] & 0x8000 ? -IQ1S_DELTA : IQ1S_DELTA;for (int l = 0; l < 4; ++l) {- const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));for (int j = 0; j < 8; ++j) {- //y[j] = db[l] * grid[l][j];- y[j] = db[l] * grid[j];+ y[j] = dl * (grid[j] + delta);}y += 8;}qs += 4;- sc += 2;}}}@@ -4252,6 +3541,9 @@ void dequantize_row_iq4_nl(const block_iq4_nl * restrict x, float * restrict y,void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y, int k) {assert(k % QK_K == 0);+#if QK_K == 64+ dequantize_row_iq4_nl((const block_iq4_nl *)x, y, k);+#elseconst int nb = k / QK_K;for (int i = 0; i < nb; i++) {@@ -4271,6 +3563,7 @@ void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y,qs += 16;}}+#endif}//===================================== Q8_K ==============================================@@ -6438,15 +5731,7 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void *#elif defined(__AVX2__)- const __m128i m4 = _mm_set1_epi8(0xf);- const __m128i m1 = _mm_set1_epi8(1);- const __m256i m511 = _mm256_set1_epi16(511);const __m256i mone = _mm256_set1_epi8(1);-- static const uint8_t k_bit_helper[32] = {- 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,- 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,- };static const char block_sign_shuffle_mask_1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,@@ -6460,11 +5745,77 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void *0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};- const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper);const __m256i bit_selector_mask = _mm256_loadu_si256((const __m256i*)bit_selector_mask_bytes);const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1);const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2);+#if QK_K == 64+ static const uint8_t k_bit_helper[16] = {+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ };+ const __m128i bit_helper = _mm_loadu_si128((const __m128i*)k_bit_helper);+ const __m128i m511 = _mm_set1_epi16(511);+ typedef union {+ __m128i vec_index;+ uint16_t index[8];+ } index_t;++ index_t idx;+ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const __m128i q2_data = _mm_loadu_si128((const __m128i*)x[i].qs);+ idx.vec_index = _mm_and_si128(q2_data, m511);++ const __m128i partial_sign_bits = _mm_srli_epi16(q2_data, 9);+ const __m128i partial_sign_bits_upper = _mm_srli_epi16(q2_data, 13);+ const __m128i partial_sign_bits_for_counting = _mm_xor_si128(partial_sign_bits, partial_sign_bits_upper);++ const __m128i odd_bits = _mm_shuffle_epi8(bit_helper, partial_sign_bits_for_counting);+ const __m128i full_sign_bits = _mm_or_si128(partial_sign_bits, odd_bits);+ const __m256i full_signs = MM256_SET_M128I(full_sign_bits, full_sign_bits);++ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)y[i].qs);+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)(y[i].qs+32));++ const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[idx.index[3]], iq2xs_grid[idx.index[2]],+ iq2xs_grid[idx.index[1]], iq2xs_grid[idx.index[0]]);+ const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[idx.index[7]], iq2xs_grid[idx.index[6]],+ iq2xs_grid[idx.index[5]], iq2xs_grid[idx.index[4]]);++ __m256i signs;+ signs = _mm256_shuffle_epi8(full_signs, block_sign_shuffle_1);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone));++ signs = _mm256_shuffle_epi8(full_signs, block_sign_shuffle_2);+ signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone));++ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);++ const __m256i sc1 = MM256_SET_M128I(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[0] & 0xf)+1));+ const __m256i sc2 = MM256_SET_M128I(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[1] & 0xf)+1));++ const __m256i sum = _mm256_add_epi32(_mm256_madd_epi16(sc1, dot1), _mm256_madd_epi16(sc2, dot2));++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sum), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);+#else++ static const uint8_t k_bit_helper[32] = {+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ };+ const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper);+ const __m256i m511 = _mm256_set1_epi16(511);+ const __m128i m4 = _mm_set1_epi8(0xf);+ const __m128i m1 = _mm_set1_epi8(1);+uint64_t aux64;// somewhat hacky, but gives a significant boost in performance@@ -6512,8 +5863,8 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void *const __m128i full_signs_l = _mm256_castsi256_si128(full_sign_bits);const __m128i full_signs_h = _mm256_extractf128_si256(full_sign_bits, 1);- const __m256i full_signs_1 = _mm256_set_m128i(full_signs_l, full_signs_l);- const __m256i full_signs_2 = _mm256_set_m128i(full_signs_h, full_signs_h);+ const __m256i full_signs_1 = MM256_SET_M128I(full_signs_l, full_signs_l);+ const __m256i full_signs_2 = MM256_SET_M128I(full_signs_h, full_signs_h);__m256i signs;signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_1);@@ -6553,6 +5904,7 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void *}*s = 0.125f * hsum_float_8(accumf);+#endif#else@@ -6615,8 +5967,8 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void *static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};- const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1);- const uint8x16_t mask2 = vld1q_u8(k_mask2);+ const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);+ const uint8x16_t mask2 = vld1q_u8(k_mask2);const uint8x16_t m1 = vdupq_n_u8(1);const int32x4_t vzero = vdupq_n_s32(0);@@ -6647,7 +5999,7 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void *vld1_s8((const int8_t *)(iq2s_grid + (qs[7] | ((qh[ib32+1] << 2) & 0x300)))));qs += 8;- vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16)));+ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);vs.val[0] = vceqq_u8(vs.val[0], mask2);@@ -6656,7 +6008,7 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void *q2s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[0]);q2s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[1]);- vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16)));+ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);vs.val[0] = vceqq_u8(vs.val[0], mask2);@@ -6727,12 +6079,12 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void *iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);qs += 8;- __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16));+ __m256i aux256 = _mm256_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);- aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16));+ aux256 = _mm256_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);@@ -6932,7 +6284,7 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void#endif}-void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);assert(nrc == 1);UNUSED(nrc);@@ -6947,18 +6299,35 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v#if defined(__ARM_NEON)+ typedef union {+ uint16x8_t vec_index;+ uint16_t index[8];+ } vec_index_t;+static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03};static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};- const uint8x16x2_t mask1 = vld1q_u8_x2(k_mask1);- const uint8x16_t mask2 = vld1q_u8(k_mask2);+ static const int16_t k_shift[8] = {8, 7, 6, 5, 4, 3, 2, 1};++ const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);+ const uint8x16_t mask2 = vld1q_u8(k_mask2);++ const int16x8_t hshift = vld1q_s16(k_shift);+ const uint16x8_t m256 = vdupq_n_u16(256);+ const uint8x16_t m1 = vdupq_n_u8(1);uint8x16x2_t vs;ggml_int8x16x4_t q3s;ggml_int8x16x4_t q8b;+ vec_index_t idx;++#if QK_K == 256+ uint32_t scales32[2];+ const uint8_t * scales8 = (const uint8_t *)scales32;+#endiffloat sumf = 0;for (int i = 0; i < nb; ++i) {@@ -6967,47 +6336,63 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const vconst uint8_t * restrict qh = x[i].qh;const uint16_t * restrict signs = (const uint16_t *)x[i].signs;const int8_t * restrict q8 = y[i].qs;++#if QK_K == 256+ memcpy(scales32, x[i].scales, 4);+ scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101;+ scales32[0] = ((scales32[0] & 0x0f0f0f0f) << 1) | 0x01010101;+#endif+int sumi1 = 0, sumi2 = 0;for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {q8b = ggml_vld1q_s8_x4(q8); q8 += 64;- const uint32x4_t aux32x4_0 = {iq3xs_grid[qs[ 0] | ((qh[ib32+0] << 8) & 256)], iq3xs_grid[qs[ 1] | ((qh[ib32+0] << 7) & 256)],- iq3xs_grid[qs[ 2] | ((qh[ib32+0] << 6) & 256)], iq3xs_grid[qs[ 3] | ((qh[ib32+0] << 5) & 256)]};- const uint32x4_t aux32x4_1 = {iq3xs_grid[qs[ 4] | ((qh[ib32+0] << 4) & 256)], iq3xs_grid[qs[ 5] | ((qh[ib32+0] << 3) & 256)],- iq3xs_grid[qs[ 6] | ((qh[ib32+0] << 2) & 256)], iq3xs_grid[qs[ 7] | ((qh[ib32+0] << 1) & 256)]};- const uint32x4_t aux32x4_2 = {iq3xs_grid[qs[ 8] | ((qh[ib32+1] << 8) & 256)], iq3xs_grid[qs[ 9] | ((qh[ib32+1] << 7) & 256)],- iq3xs_grid[qs[10] | ((qh[ib32+1] << 6) & 256)], iq3xs_grid[qs[11] | ((qh[ib32+1] << 5) & 256)]};- const uint32x4_t aux32x4_3 = {iq3xs_grid[qs[12] | ((qh[ib32+1] << 4) & 256)], iq3xs_grid[qs[13] | ((qh[ib32+1] << 3) & 256)],- iq3xs_grid[qs[14] | ((qh[ib32+1] << 2) & 256)], iq3xs_grid[qs[15] | ((qh[ib32+1] << 1) & 256)]};- qs += 16;- vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | (signs[1] << 16)));+ const uint8x16_t idx_l = vld1q_u8(qs); qs += 16;+ idx.vec_index = vorrq_u16(vmovl_u8(vget_low_u8 (idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+0]), hshift), m256));+ const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],+ iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);+ const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],+ iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);+ idx.vec_index = vorrq_u16(vmovl_u8(vget_high_u8(idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+1]), hshift), m256));+ const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],+ iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);+ const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],+ iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);+++ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);- vs.val[0] = vceqq_u8(vs.val[0], mask2);- vs.val[1] = vceqq_u8(vs.val[1], mask2);+ vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);+ vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);- q3s.val[0] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[0], vreinterpretq_u8_u32(aux32x4_0))), vreinterpretq_s8_u8(vs.val[0]));- q3s.val[1] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[1], vreinterpretq_u8_u32(aux32x4_1))), vreinterpretq_s8_u8(vs.val[1]));+ q3s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_0));+ q3s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_1));- vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | (signs[3] << 16)));+ vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);- vs.val[0] = vceqq_u8(vs.val[0], mask2);- vs.val[1] = vceqq_u8(vs.val[1], mask2);+ vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);+ vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);signs += 4;- q3s.val[2] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[0], vreinterpretq_u8_u32(aux32x4_2))), vreinterpretq_s8_u8(vs.val[0]));- q3s.val[3] = vsubq_s8(vreinterpretq_s8_u8(veorq_u8(vs.val[1], vreinterpretq_u8_u32(aux32x4_3))), vreinterpretq_s8_u8(vs.val[1]));+ q3s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_2));+ q3s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_3));const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);+#if QK_K == 256+ sumi1 += vaddvq_s32(p1) * scales8[ib32/2+0];+ sumi2 += vaddvq_s32(p2) * scales8[ib32/2+4];+#elsesumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32/2] & 0xf));sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32/2] >> 4));+#endif}sumf += d*(sumi1 + sumi2);}- *s = 0.25f * sumf;+ *s = sumf;#elif defined(__AVX2__)@@ -7022,6 +6407,16 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const vconst __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);+ const __m256i idx_shift = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);+ const __m256i idx_mask = _mm256_set1_epi32(256);++ typedef union {+ __m256i vec[2];+ uint32_t index[16];+ } index_t;++ index_t idx;+__m256 accumf = _mm256_setzero_ps();for (int i = 0; i < nb; ++i) {const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;@@ -7034,24 +6429,25 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const vfor (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- const __m256i q2_1 = _mm256_set_epi32(iq3xs_grid[qs[7] | ((qh[ib32+0] << 1) & 256)],- iq3xs_grid[qs[6] | ((qh[ib32+0] << 2) & 256)],- iq3xs_grid[qs[5] | ((qh[ib32+0] << 3) & 256)],- iq3xs_grid[qs[4] | ((qh[ib32+0] << 4) & 256)],- iq3xs_grid[qs[3] | ((qh[ib32+0] << 5) & 256)],- iq3xs_grid[qs[2] | ((qh[ib32+0] << 6) & 256)],- iq3xs_grid[qs[1] | ((qh[ib32+0] << 7) & 256)],- iq3xs_grid[qs[0] | ((qh[ib32+0] << 8) & 256)]);- qs += 8;- const __m256i q2_2 = _mm256_set_epi32(iq3xs_grid[qs[7] | ((qh[ib32+1] << 1) & 256)],- iq3xs_grid[qs[6] | ((qh[ib32+1] << 2) & 256)],- iq3xs_grid[qs[5] | ((qh[ib32+1] << 3) & 256)],- iq3xs_grid[qs[4] | ((qh[ib32+1] << 4) & 256)],- iq3xs_grid[qs[3] | ((qh[ib32+1] << 5) & 256)],- iq3xs_grid[qs[2] | ((qh[ib32+1] << 6) & 256)],- iq3xs_grid[qs[1] | ((qh[ib32+1] << 7) & 256)],- iq3xs_grid[qs[0] | ((qh[ib32+1] << 8) & 256)]);- qs += 8;+ const __m256i idx_l = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)qs)); qs += 16;+ idx.vec[0] = _mm256_set1_epi32(qh[ib32+0]);+ idx.vec[1] = _mm256_set1_epi32(qh[ib32+1]);+ idx.vec[0] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[0], idx_shift), idx_mask);+ idx.vec[1] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[1], idx_shift), idx_mask);+ idx.vec[0] = _mm256_or_si256(idx.vec[0], _mm256_cvtepi16_epi32(_mm256_castsi256_si128(idx_l)));+ idx.vec[1] = _mm256_or_si256(idx.vec[1], _mm256_cvtepi16_epi32(_mm256_extractf128_si256(idx_l, 1)));++ // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.+ //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);+ //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);+ const __m256i q2_1 = _mm256_set_epi32(+ iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],+ iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]+ );+ const __m256i q2_2 = _mm256_set_epi32(+ iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],+ iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]+ );__m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16));aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);@@ -7079,7 +6475,7 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v}- *s = 0.25f * hsum_float_8(accumf);+ *s = hsum_float_8(accumf);#else@@ -7096,8 +6492,8 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const vconst uint32_t ls2 = 2*(x[i].scales[ib32/2] >> 4) + 1;int32_t sumi = 0;for (int l = 0; l < 4; ++l) {- const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256)));- const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256)));+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256)));for (int j = 0; j < 4; ++j) {sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);@@ -7109,8 +6505,8 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const vbsum += sumi * ls1;sumi = 0;for (int l = 0; l < 4; ++l) {- const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256)));- const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256)));+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256)));for (int j = 0; j < 4; ++j) {sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);@@ -7123,10 +6519,11 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const v}sumf += d * bsum;}- *s = 0.25f * sumf;+ *s = sumf;#endif}+#ifdef __AVX2__static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {const __m256i ax = _mm256_sign_epi8(x, x);@@ -7135,7 +6532,7 @@ static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {}#endif-void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {assert(n % QK_K == 0);assert(nrc == 1);UNUSED(nrc);@@ -7150,147 +6547,117 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const#if defined __ARM_NEON- const uint8x16_t m8 = vdupq_n_u8(0x08);- const uint8x16_t m7 = vdupq_n_u8(0x07);- const uint8x16_t m1 = vdupq_n_u8(0x01);- const int32x4_t vzero = vdupq_n_s32(0);-- uint16_t gindex[8];- uint16x8x2_t vindex;- int8x16x4_t q1b;- int8x16x4_t q8b;- uint16x8x4_t scales;- int32x4x2_t sumi;- int32x4x2_t dotq;+ ggml_int8x16x4_t q1b;+ ggml_int8x16x4_t q8b;float sumf = 0;for (int i = 0; i < nb; ++i) {- const int8_t * q8 = y[i].qs;- const uint8_t * qs = x[i].qs;- const uint8_t * sc = x[i].scales;+ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;- sumi.val[0] = sumi.val[1] = vzero;+ int sumi1 = 0, sumi2 = 0, sumi3 = 0;- for (int i128 = 0; i128 < QK_K/128; ++i128) {- const uint8x16_t ql = vld1q_u8(qs); qs += 16;- const uint8x8_t tm1 = vld1_u8 (sc); sc += 8;- const uint8x8_t tm2 = vshr_n_u8(tm1, 4);- const uint8x16_t qh = vcombine_u8(vzip1_u8(tm1, tm2), vzip2_u8(tm1, tm2));- const uint8x16_t hbit = vandq_u8(qh, m8);- vindex.val[0] = vorrq_u16(vmovl_u8(vget_low_u8 (ql)), vshlq_n_u16(vmovl_u8(vget_low_u8 (hbit)), 5));- vindex.val[1] = vorrq_u16(vmovl_u8(vget_high_u8(ql)), vshlq_n_u16(vmovl_u8(vget_high_u8(hbit)), 5));- const uint8x16_t scales8 = vorrq_u8(vshlq_n_u8(vandq_u8(qh, m7), 1), m1);- scales.val[0] = vmovl_u8(vget_low_u8 (scales8));- scales.val[1] = vmovl_u8(vget_high_u8 (scales8));+ for (int ib = 0; ib < QK_K/32; ib += 2) {- for (int l = 0; l < 2; ++l) {- vst1q_u16(gindex+0, vindex.val[l]);- q1b.val[0] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[0])), vld1_s8((const void *)(iq1s_grid+gindex[1])));- q1b.val[1] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[2])), vld1_s8((const void *)(iq1s_grid+gindex[3])));- q1b.val[2] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[4])), vld1_s8((const void *)(iq1s_grid+gindex[5])));- q1b.val[3] = vcombine_s8(vld1_s8((const void *)(iq1s_grid+gindex[6])), vld1_s8((const void *)(iq1s_grid+gindex[7])));- q8b = ggml_vld1q_s8_x4(q8); q8 += 64;+ q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[ib+0] << 8) & 0x700)))),+ vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[ib+0] << 5) & 0x700)))));+ q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[ib+0] << 2) & 0x700)))),+ vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[ib+0] >> 1) & 0x700)))));+ q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[ib+1] << 8) & 0x700)))),+ vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[ib+1] << 5) & 0x700)))));+ q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[ib+1] << 2) & 0x700)))),+ vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[ib+1] >> 1) & 0x700)))));+ qs += 8;- dotq.val[0] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[0], q8b.val[0]), ggml_vdotq_s32(vzero, q1b.val[1], q8b.val[1]));- dotq.val[1] = vpaddq_s32(ggml_vdotq_s32(vzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(vzero, q1b.val[3], q8b.val[3]));+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;++ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[0], q8b.val[0]), q1b.val[1], q8b.val[1]);+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[2], q8b.val[2]), q1b.val[3], q8b.val[3]);++ const int ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;+ const int ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;+ sumi1 += vaddvq_s32(p1) * ls1;+ sumi2 += vaddvq_s32(p2) * ls2;+ sumi3 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * ls1 * (qh[ib+0] & 0x8000 ? -1 : 1)+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * ls2 * (qh[ib+1] & 0x8000 ? -1 : 1);- sumi.val[0] = vmlaq_s32(sumi.val[0], dotq.val[0], vreinterpretq_s32_u32(vmovl_u16(vget_low_u16 (scales.val[l]))));- sumi.val[1] = vmlaq_s32(sumi.val[1], dotq.val[1], vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales.val[l]))));- }}- sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * vaddvq_s32(vaddq_s32(sumi.val[0], sumi.val[1]));+ sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * (sumi1 + sumi2 + IQ1S_DELTA * sumi3);}*s = sumf;#elif defined __AVX2__- const __m128i m8 = _mm_set1_epi8(0x08);- const __m128i m7 = _mm_set1_epi8(0x07);- const __m128i m1 = _mm_set1_epi8(0x01);- const __m128i shuffle_h = _mm_set_epi8(15, 7, 14, 6, 13, 5, 12, 4, 11, 3, 10, 2, 9, 1, 8, 0);- const __m128i shuffle_s[4] = {- _mm_set_epi32(0x03030303, 0x02020202, 0x01010101, 0x00000000),- _mm_set_epi32(0x07070707, 0x06060606, 0x05050505, 0x04040404),- _mm_set_epi32(0x0b0b0b0b, 0x0a0a0a0a, 0x09090909, 0x08080808),- _mm_set_epi32(0x0f0f0f0f, 0x0e0e0e0e, 0x0d0d0d0d, 0x0c0c0c0c)- };-- uint64_t aux64;-- __m256i v_gindex;- const uint16_t * gindex = (const uint16_t *)&v_gindex;-__m256 accum = _mm256_setzero_ps();+ float accum1 = 0;for (int i = 0; i < nb; ++i) {- const int8_t * q8 = y[i].qs;- const uint8_t * qs = x[i].qs;- const uint8_t * sc = x[i].scales;+ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;__m256i sumi = _mm256_setzero_si256();- for (int i128 = 0; i128 < QK_K/128; ++i128) {- const __m128i ql = _mm_loadu_si128((const __m128i*)qs); qs += 16;- memcpy(&aux64, sc, 8); sc += 8;- const __m128i qh = _mm_shuffle_epi8(_mm_set_epi64x(aux64 >> 4, aux64), shuffle_h);- const __m256i hbit = _mm256_cvtepu8_epi16(_mm_and_si128(qh, m8));- v_gindex = _mm256_or_si256(_mm256_cvtepu8_epi16(ql), _mm256_slli_epi16(hbit, 5));- const __m128i scales = _mm_or_si128(_mm_slli_epi16(_mm_and_si128(qh, m7), 1), m1);+ int sumi1 = 0;+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)],+ iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);+ const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)],+ iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);+ qs += 8;+ const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- for (int i32 = 0; i32 < 4; ++i32) {- const __m256i q8b = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q1b = _mm256_set_epi64x(iq1s_grid[gindex[4*i32+3]], iq1s_grid[gindex[4*i32+2]],- iq1s_grid[gindex[4*i32+1]], iq1s_grid[gindex[4*i32+0]]);- const __m256i dot = mul_add_epi8(q1b, q8b);- const __m256i s16 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, shuffle_s[i32]));- const __m256i p = _mm256_madd_epi16(s16, dot);- sumi = _mm256_add_epi32(sumi, p);- }+ const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);+ const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);+ const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;+ const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(ls1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(ls2));+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p1, p2));+ sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;}- accum = _mm256_fmadd_ps(_mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(x[i].d)), _mm256_cvtepi32_ps(sumi), accum);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ accum = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), accum);+ accum1 += d * sumi1;}- *s = hsum_float_8(accum);+ *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;#else- int db[4];- uint16_t idx[4];-float sumf = 0;- for (int i = 0; i < nb; ++i) {+ for (int i = 0; i < nb; i++) {- const int8_t * q8 = y[i].qs;- const uint8_t * qs = x[i].qs;- const uint8_t * sc = x[i].scales;+ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;- int sumi = 0;- for (int i32 = 0; i32 < QK_K/32; ++i32) {- idx[0] = qs[0] | ((sc[0] & 0x08) << 5);- idx[1] = qs[1] | ((sc[0] & 0x80) << 1);- idx[2] = qs[2] | ((sc[1] & 0x08) << 5);- idx[3] = qs[3] | ((sc[1] & 0x80) << 1);- db[0] = (2*(sc[0] & 7) + 1);- db[1] = (2*((sc[0] >> 4) & 7) + 1);- db[2] = (2*(sc[1] & 7) + 1);- db[3] = (2*((sc[1] >> 4) & 7) + 1);+ int sumi = 0, sumi1 = 0;+ for (int ib = 0; ib < QK_K/32; ++ib) {+ const int ls = 2*((qh[ib] >> 12) & 7) + 1;+ const int delta = qh[ib] & 0x8000 ? -1 : 1;+ int lsum = 0;for (int l = 0; l < 4; ++l) {- const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]);- int suml = 0;- for (int j = 0; j < 8; ++j) suml += q8[j] * grid[j];- sumi += db[l] * suml;+ const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));+ for (int j = 0; j < 8; ++j) {+ lsum += q8[j] * grid[j];+ }q8 += 8;}+ sumi += ls * lsum;+ sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]);qs += 4;- sc += 2;}- sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * sumi;+ sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);}*s = sumf;@@ -7331,16 +6698,17 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void *q8b.val[2] = vld1q_s8(y[ib+1].qs);q8b.val[3] = vld1q_s8(y[ib+1].qs + 16);- q4b.val[0] = vqtbl1q_s8(values, vandq_u8(q4bits.val[0], m4b));- q4b.val[1] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));- q4b.val[2] = vqtbl1q_s8(values, vandq_u8(q4bits.val[1], m4b));- q4b.val[3] = vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));+ q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[0], m4b));+ q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));+ q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits.val[1], m4b));+ q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);- sumf += (float)x[ib+0].d * (float)y[ib+0].d * vaddvq_s32(prod_1) + (float)x[ib+1].d * (float)y[ib+1].d * vaddvq_s32(prod_2);-+ sumf +=+ GGML_FP16_TO_FP32(x[ib+0].d) * GGML_FP16_TO_FP32(y[ib+0].d) * vaddvq_s32(prod_1) ++ GGML_FP16_TO_FP32(x[ib+1].d) * GGML_FP16_TO_FP32(y[ib+1].d) * vaddvq_s32(prod_2);}*s = sumf;@@ -7358,10 +6726,10 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void *const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[1].qs);const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[0].qs);const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[1].qs);- const __m256i q4b_1 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),- _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));- const __m256i q4b_2 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),- _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));+ const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));+ const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);@@ -7399,6 +6767,9 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void *UNUSED(by);UNUSED(bs);assert(n % QK_K == 0);+#if QK_K == 64+ ggml_vec_dot_iq4_nl_q8_0(n, s, bs, vx, bx, vy, by, nrc);+#elseconst block_iq4_xs * restrict x = vx;const block_q8_K * restrict y = vy;@@ -7465,10 +6836,10 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void *const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)qs); qs += 16;const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- const __m256i q4b_1 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),- _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));- const __m256i q4b_2 = _mm256_set_m128i(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),- _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));+ const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));+ const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),+ _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;@@ -7518,6 +6889,7 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void *}*s = sumf;#endif+#endif}// ================================ IQ2 quantization =============================================@@ -7528,22 +6900,25 @@ typedef struct {uint16_t * neighbours;} iq2_entry_t;-static iq2_entry_t iq2_data[3] = {+static iq2_entry_t iq2_data[4] = {+ {NULL, NULL, NULL},{NULL, NULL, NULL},{NULL, NULL, NULL},{NULL, NULL, NULL},};static inline int iq2_data_index(enum ggml_type type) {- GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S);+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ2_S);return type == GGML_TYPE_IQ2_XXS ? 0 :- type == GGML_TYPE_IQ2_XS ? 1 : 2;+ type == GGML_TYPE_IQ2_XS ? 1 :+ type == GGML_TYPE_IQ1_S ? 2 : 3;}static inline int iq2_grid_size(enum ggml_type type) {- GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S);+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ2_S);return type == GGML_TYPE_IQ2_XXS ? 256 :- type == GGML_TYPE_IQ2_XS ? 512 : 512;+ type == GGML_TYPE_IQ2_XS ? 512 :+ type == GGML_TYPE_IQ1_S ? NGRID_IQ1S : 1024;}static int iq2_compare_func(const void * left, const void * right) {@@ -7610,50 +6985,214 @@ void iq2xs_init_impl(enum ggml_type type) {40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048,42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690,};- static const uint16_t kgrid_1bit_512[512] = {- 10, 33, 41, 85, 132, 134, 160, 162, 277, 337, 340, 345, 357, 405, 516, 545,- 553, 598, 641, 650, 681, 1042, 1044, 1097, 1169, 1176, 1320, 1345, 1365, 1378, 1434, 1444,- 1545, 1617, 1642, 1685, 2053, 2080, 2089, 2133, 2176, 2182, 2208, 2214, 2306, 2384, 2393, 2440,- 2453, 2581, 2664, 2690, 2721, 4117, 4161, 4182, 4184, 4261, 4357, 4369, 4372, 4377, 4390, 4422,- 4432, 4437, 4449, 4457, 4485, 4497, 4505, 4629, 4677, 4696, 4774, 5205, 5217, 5225, 5386, 5397,- 5409, 5445, 5457, 5460, 5461, 5462, 5465, 5472, 5477, 5525, 5545, 5650, 5668, 5717, 5729, 5769,- 5777, 6212, 6234, 6244, 6293, 6424, 6482, 6485, 6502, 6505, 6529, 6538, 6565, 6656, 6682, 6788,- 6806, 6820, 8218, 8224, 8226, 8232, 8277, 8326, 8354, 8469, 8521, 8530, 8549, 8596, 8737, 8794,- 9221, 9253, 9348, 9369, 9380, 9474, 9557, 9633, 9732, 9753, 9793, 9830, 9862, 9880, 10240, 10272,- 10282, 10321, 10406, 10517, 10530, 10566, 10585, 10645, 10896, 16466, 16468, 16473, 16485, 16646, 16660, 16665,- 16725, 16793, 16806, 16914, 16969, 16977, 16996, 17028, 17057, 17408, 17416, 17434, 17493, 17512, 17578, 17685,- 17696, 17733, 17745, 17748, 17749, 17750, 17753, 17765, 17794, 17813, 17946, 17984, 18005, 18072, 18453, 18529,- 18569, 18722, 18756, 18762, 18773, 18794, 18833, 18853, 18945, 19026, 19033, 19077, 20489, 20497, 20500, 20517,- 20565, 20586, 20610, 20633, 20757, 20769, 20776, 20805, 20817, 20820, 20821, 20822, 20825, 20837, 20864, 20872,- 20885, 20896, 21002, 21029, 21077, 21146, 21510, 21525, 21573, 21585, 21588, 21589, 21590, 21593, 21605, 21653,- 21665, 21765, 21777, 21780, 21781, 21782, 21785, 21797, 21825, 21828, 21829, 21830, 21833, 21840, 21841, 21842,- 21844, 21846, 21848, 21849, 21850, 21857, 21860, 21861, 21862, 21865, 21893, 21905, 21908, 21909, 21910, 21913,- 21925, 22024, 22037, 22085, 22097, 22100, 22101, 22102, 22105, 22117, 22165, 22545, 22566, 22568, 22594, 22608,- 22613, 22676, 22697, 22793, 22805, 22853, 22865, 22868, 22869, 22870, 22873, 22885, 22933, 22946, 23046, 23072,- 23125, 23209, 24597, 24640, 24665, 24673, 24725, 24833, 24840, 24869, 24917, 24934, 24965, 25001, 25108, 25110,- 25152, 25184, 25192, 25234, 25616, 25618, 25625, 25685, 25704, 25738, 25744, 25770, 25877, 25897, 25925, 25937,- 25940, 25941, 25942, 25945, 25957, 25986, 26005, 26186, 26197, 26276, 26632, 26634, 26725, 26757, 26770, 26885,- 26965, 26976, 26986, 27032, 27153, 27174, 27200, 27208, 27240, 27269, 27282, 27290, 32778, 32800, 32802, 32808,- 32810, 32853, 32904, 32922, 32930, 32932, 33105, 33110, 33112, 33125, 33157, 33280, 33288, 33301, 33312, 33320,- 33424, 33797, 33829, 33858, 34068, 34133, 34146, 34176, 34217, 34306, 34342, 34441, 34454, 34468, 34832, 34918,- 34965, 34984, 35094, 35137, 35161, 35208, 35232, 35332, 35338, 35368, 35429, 36932, 36934, 36953, 37009, 37125,- 37136, 37138, 37145, 37157, 37205, 37220, 37258, 37290, 37444, 37446, 37465, 37478, 37525, 37905, 37968, 37973,- 38040, 38054, 38145, 38154, 38165, 38180, 38186, 38213, 38225, 38228, 38229, 38230, 38233, 38245, 38293, 38485,- 38504, 38530, 38938, 38985, 38993, 39012, 39040, 39173, 39192, 39253, 39265, 39301, 39316, 39322, 39442, 39497,- 39504, 39590, 40970, 40984, 40992, 41002, 41045, 41120, 41128, 41237, 41289, 41297, 41317, 41364, 41366, 41514,- 41557, 41633, 41989, 42021, 42056, 42068, 42074, 42113, 42242, 42265, 42274, 42325, 42340, 42402, 42501, 42512,- 42533, 42624, 42632, 42666, 43040, 43093, 43106, 43168, 43176, 43264, 43286, 43345, 43429, 43590, 43618, 43680,+ static const uint16_t kgrid_1bit_2048[NGRID_IQ1S] = {+ 0, 2, 5, 8, 10, 17, 21, 32, 34, 40, 42, 69, 81, 84, 86, 101,+ 128, 130, 136, 138, 149, 160, 162, 168, 170, 260, 261, 273, 276, 278, 281, 282,+ 293, 321, 326, 329, 338, 341, 346, 353, 356, 358, 360, 389, 401, 404, 406, 421,+ 512, 514, 520, 522, 533, 544, 546, 552, 554, 581, 593, 601, 612, 617, 640, 642,+ 648, 650, 657, 661, 665, 672, 674, 680, 682, 1041, 1044, 1046, 1061, 1089, 1097, 1109,+ 1114, 1124, 1125, 1169, 1177, 1189, 1281, 1284, 1285, 1286, 1301, 1304, 1306, 1321, 1344, 1349,+ 1354, 1360, 1361, 1364, 1365, 1366, 1369, 1376, 1378, 1381, 1384, 1386, 1409, 1425, 1429, 1432,+ 1434, 1441, 1444, 1445, 1446, 1449, 1556, 1561, 1601, 1604, 1616, 1618, 1621, 1624, 1632, 1633,+ 1638, 1641, 1669, 1681, 1684, 1689, 2048, 2050, 2056, 2058, 2069, 2080, 2082, 2088, 2090, 2117,+ 2129, 2134, 2149, 2176, 2178, 2184, 2186, 2197, 2208, 2210, 2216, 2218, 2309, 2321, 2324, 2329,+ 2340, 2341, 2369, 2384, 2385, 2389, 2401, 2404, 2409, 2449, 2452, 2454, 2457, 2469, 2560, 2562,+ 2568, 2570, 2581, 2592, 2594, 2600, 2602, 2629, 2641, 2649, 2657, 2661, 2688, 2690, 2693, 2696,+ 2698, 2709, 2720, 2722, 2728, 2730, 4112, 4113, 4116, 4121, 4132, 4133, 4161, 4164, 4176, 4181,+ 4184, 4193, 4196, 4197, 4201, 4241, 4244, 4246, 4257, 4261, 4353, 4356, 4358, 4361, 4368, 4370,+ 4373, 4376, 4385, 4388, 4393, 4421, 4426, 4432, 4433, 4434, 4436, 4437, 4438, 4441, 4448, 4453,+ 4484, 4498, 4501, 4513, 4516, 4625, 4628, 4630, 4645, 4672, 4678, 4681, 4690, 4693, 4696, 4698,+ 4708, 4710, 4741, 4753, 4756, 4758, 4773, 5121, 5126, 5129, 5140, 5141, 5144, 5145, 5153, 5158,+ 5185, 5189, 5190, 5192, 5194, 5201, 5204, 5205, 5206, 5209, 5218, 5221, 5224, 5252, 5257, 5264,+ 5268, 5269, 5272, 5273, 5274, 5281, 5284, 5285, 5289, 5378, 5381, 5386, 5393, 5396, 5397, 5398,+ 5401, 5408, 5410, 5413, 5416, 5418, 5441, 5444, 5445, 5446, 5457, 5458, 5460, 5461, 5462, 5465,+ 5466, 5473, 5476, 5477, 5478, 5481, 5504, 5506, 5508, 5509, 5512, 5514, 5520, 5521, 5524, 5525,+ 5526, 5529, 5530, 5536, 5538, 5541, 5633, 5636, 5637, 5638, 5653, 5654, 5656, 5658, 5665, 5670,+ 5696, 5698, 5700, 5701, 5704, 5706, 5713, 5717, 5718, 5720, 5721, 5729, 5732, 5733, 5736, 5737,+ 5738, 5766, 5770, 5778, 5781, 5796, 5801, 6161, 6166, 6181, 6209, 6212, 6214, 6217, 6224, 6229,+ 6232, 6234, 6240, 6241, 6244, 6246, 6249, 6277, 6289, 6292, 6309, 6416, 6418, 6421, 6426, 6433,+ 6437, 6466, 6468, 6469, 6472, 6481, 6484, 6485, 6486, 6489, 6490, 6496, 6501, 6506, 6537, 6545,+ 6546, 6549, 6552, 6561, 6566, 6569, 6665, 6678, 6692, 6694, 6724, 6726, 6729, 6736, 6738, 6741,+ 6744, 6753, 6758, 6761, 6789, 6801, 6806, 6810, 8192, 8194, 8200, 8202, 8213, 8224, 8226, 8229,+ 8232, 8234, 8261, 8273, 8281, 8289, 8293, 8320, 8322, 8328, 8330, 8341, 8352, 8354, 8357, 8360,+ 8362, 8453, 8465, 8468, 8473, 8485, 8514, 8516, 8521, 8533, 8536, 8538, 8545, 8548, 8549, 8550,+ 8581, 8592, 8598, 8601, 8613, 8705, 8712, 8714, 8721, 8725, 8736, 8738, 8744, 8746, 8773, 8785,+ 8790, 8793, 8805, 8833, 8840, 8842, 8849, 8853, 8864, 8866, 8872, 8874, 9221, 9236, 9238, 9241,+ 9253, 9284, 9285, 9286, 9289, 9298, 9301, 9304, 9306, 9318, 9349, 9361, 9364, 9369, 9377, 9381,+ 9481, 9493, 9505, 9513, 9536, 9541, 9544, 9553, 9556, 9557, 9561, 9570, 9573, 9576, 9609, 9616,+ 9620, 9621, 9624, 9626, 9633, 9636, 9638, 9641, 9733, 9744, 9746, 9753, 9765, 9793, 9801, 9813,+ 9824, 9825, 9833, 9860, 9862, 9872, 9882, 10240, 10242, 10248, 10250, 10261, 10272, 10274, 10280, 10282,+ 10309, 10321, 10324, 10341, 10368, 10370, 10376, 10378, 10400, 10402, 10408, 10410, 10505, 10513, 10516, 10521,+ 10533, 10566, 10569, 10578, 10581, 10593, 10596, 10598, 10601, 10629, 10640, 10646, 10649, 10660, 10661, 10752,+ 10754, 10760, 10762, 10784, 10786, 10792, 10794, 10821, 10833, 10838, 10841, 10853, 10880, 10882, 10888, 10890,+ 10901, 10912, 10914, 10920, 10922, 16389, 16401, 16406, 16421, 16457, 16466, 16469, 16472, 16474, 16481, 16484,+ 16486, 16532, 16537, 16545, 16550, 16640, 16641, 16644, 16646, 16649, 16658, 16661, 16662, 16664, 16666, 16673,+ 16678, 16681, 16709, 16712, 16714, 16721, 16724, 16725, 16726, 16729, 16730, 16741, 16744, 16746, 16769, 16772,+ 16774, 16784, 16786, 16789, 16800, 16801, 16802, 16901, 16913, 16916, 16918, 16933, 16961, 16978, 16981, 16986,+ 16996, 17001, 17033, 17044, 17061, 17409, 17429, 17433, 17449, 17477, 17480, 17482, 17489, 17492, 17493, 17494,+ 17505, 17506, 17509, 17512, 17514, 17537, 17542, 17545, 17552, 17554, 17557, 17568, 17569, 17577, 17665, 17666,+ 17669, 17674, 17681, 17684, 17685, 17686, 17689, 17696, 17701, 17706, 17729, 17732, 17733, 17734, 17737, 17744,+ 17745, 17748, 17749, 17750, 17752, 17753, 17761, 17764, 17765, 17766, 17769, 17794, 17796, 17797, 17800, 17809,+ 17812, 17813, 17814, 17817, 17818, 17829, 17832, 17834, 17921, 17925, 17929, 17940, 17941, 17944, 17946, 17953,+ 17956, 17961, 17984, 17986, 17989, 17992, 18000, 18001, 18002, 18005, 18006, 18009, 18018, 18021, 18024, 18049,+ 18053, 18058, 18068, 18069, 18081, 18084, 18086, 18437, 18449, 18453, 18458, 18469, 18498, 18505, 18512, 18517,+ 18520, 18529, 18532, 18534, 18537, 18565, 18577, 18580, 18582, 18585, 18597, 18689, 18693, 18694, 18698, 18704,+ 18708, 18709, 18712, 18721, 18724, 18726, 18752, 18757, 18762, 18769, 18770, 18772, 18773, 18774, 18777, 18784,+ 18786, 18789, 18790, 18794, 18822, 18825, 18834, 18837, 18838, 18840, 18849, 18852, 18854, 18857, 18966, 19012,+ 19014, 19017, 19029, 19032, 19034, 19044, 19049, 19092, 19109, 20481, 20484, 20485, 20486, 20489, 20498, 20501,+ 20506, 20513, 20516, 20521, 20544, 20549, 20552, 20561, 20564, 20565, 20566, 20569, 20581, 20584, 20614, 20617,+ 20629, 20632, 20640, 20641, 20646, 20649, 20741, 20744, 20745, 20746, 20753, 20756, 20757, 20758, 20760, 20761,+ 20768, 20773, 20774, 20776, 20778, 20801, 20804, 20805, 20806, 20809, 20816, 20817, 20818, 20820, 20821, 20822,+ 20824, 20825, 20826, 20833, 20836, 20837, 20838, 20841, 20866, 20869, 20881, 20884, 20885, 20886, 20889, 20896,+ 20901, 20906, 20993, 20998, 21010, 21013, 21018, 21025, 21028, 21058, 21061, 21066, 21073, 21076, 21077, 21078,+ 21081, 21090, 21093, 21125, 21136, 21138, 21141, 21145, 21146, 21156, 21508, 21509, 21521, 21524, 21525, 21526,+ 21528, 21529, 21537, 21541, 21544, 21546, 21569, 21572, 21573, 21574, 21577, 21578, 21584, 21585, 21588, 21589,+ 21590, 21592, 21593, 21594, 21601, 21602, 21604, 21605, 21606, 21609, 21632, 21640, 21642, 21649, 21652, 21653,+ 21654, 21657, 21665, 21668, 21669, 21674, 21761, 21762, 21764, 21765, 21766, 21769, 21776, 21777, 21778, 21780,+ 21781, 21782, 21785, 21786, 21793, 21796, 21797, 21798, 21801, 21824, 21825, 21826, 21828, 21829, 21830, 21832,+ 21833, 21840, 21841, 21842, 21844, 21845, 21846, 21848, 21849, 21850, 21856, 21857, 21860, 21861, 21862, 21864,+ 21865, 21866, 21889, 21892, 21893, 21897, 21898, 21904, 21905, 21908, 21909, 21910, 21912, 21913, 21921, 21924,+ 21925, 21926, 21929, 22016, 22017, 22018, 22020, 22022, 22024, 22025, 22033, 22036, 22037, 22040, 22041, 22048,+ 22049, 22050, 22052, 22053, 22054, 22056, 22057, 22081, 22085, 22086, 22088, 22089, 22090, 22096, 22097, 22098,+ 22100, 22101, 22102, 22104, 22105, 22106, 22113, 22116, 22117, 22121, 22146, 22149, 22150, 22152, 22153, 22154,+ 22161, 22165, 22170, 22178, 22181, 22182, 22184, 22185, 22532, 22533, 22534, 22537, 22544, 22549, 22552, 22561,+ 22570, 22597, 22600, 22602, 22609, 22612, 22613, 22614, 22616, 22617, 22624, 22626, 22628, 22629, 22658, 22665,+ 22672, 22674, 22677, 22680, 22689, 22697, 22785, 22786, 22789, 22794, 22801, 22804, 22805, 22806, 22809, 22821,+ 22849, 22852, 22853, 22854, 22857, 22864, 22865, 22866, 22868, 22869, 22870, 22872, 22873, 22874, 22881, 22884,+ 22885, 22886, 22889, 22913, 22917, 22921, 22929, 22932, 22933, 22934, 22936, 22937, 22949, 23044, 23048, 23061,+ 23066, 23072, 23077, 23078, 23081, 23109, 23112, 23113, 23121, 23125, 23126, 23128, 23129, 23138, 23141, 23144,+ 23146, 23169, 23178, 23186, 23189, 23190, 23192, 23194, 23201, 24581, 24596, 24598, 24601, 24613, 24644, 24656,+ 24661, 24662, 24664, 24666, 24673, 24676, 24678, 24681, 24705, 24726, 24741, 24833, 24836, 24838, 24841, 24850,+ 24853, 24865, 24866, 24870, 24873, 24901, 24905, 24913, 24917, 24918, 24921, 24933, 24934, 24938, 24964, 24970,+ 24978, 24981, 24993, 24998, 25001, 25105, 25110, 25113, 25152, 25153, 25158, 25173, 25174, 25176, 25184, 25221,+ 25233, 25238, 25253, 25617, 25618, 25621, 25622, 25626, 25633, 25638, 25641, 25664, 25666, 25669, 25672, 25674,+ 25681, 25684, 25685, 25686, 25689, 25690, 25696, 25698, 25701, 25732, 25733, 25737, 25744, 25746, 25748, 25749,+ 25750, 25752, 25754, 25761, 25764, 25769, 25861, 25864, 25866, 25873, 25877, 25878, 25881, 25924, 25925, 25926,+ 25929, 25936, 25937, 25940, 25941, 25942, 25945, 25953, 25956, 25957, 25958, 25961, 25990, 25993, 25994, 26001,+ 26005, 26006, 26009, 26010, 26018, 26021, 26022, 26024, 26114, 26121, 26133, 26144, 26150, 26152, 26153, 26176,+ 26181, 26184, 26186, 26193, 26196, 26197, 26198, 26200, 26202, 26208, 26213, 26216, 26240, 26242, 26245, 26250,+ 26260, 26262, 26264, 26265, 26272, 26276, 26278, 26282, 26646, 26649, 26661, 26689, 26706, 26709, 26714, 26721,+ 26729, 26757, 26769, 26776, 26790, 26881, 26884, 26896, 26901, 26913, 26916, 26918, 26921, 26944, 26945, 26949,+ 26950, 26952, 26961, 26964, 26965, 26966, 26969, 26976, 26981, 26986, 27010, 27012, 27018, 27029, 27041, 27044,+ 27045, 27049, 27153, 27158, 27160, 27201, 27204, 27209, 27216, 27221, 27224, 27226, 27236, 27237, 27241, 27270,+ 27284, 27288, 27290, 27302, 32768, 32770, 32776, 32778, 32800, 32802, 32808, 32810, 32837, 32848, 32849, 32852,+ 32854, 32857, 32869, 32896, 32898, 32904, 32906, 32917, 32928, 32930, 32936, 32938, 33029, 33041, 33044, 33046,+ 33049, 33061, 33089, 33092, 33097, 33104, 33106, 33109, 33110, 33112, 33113, 33124, 33126, 33129, 33157, 33161,+ 33172, 33174, 33177, 33189, 33280, 33282, 33288, 33290, 33301, 33312, 33314, 33320, 33322, 33361, 33364, 33369,+ 33381, 33408, 33410, 33416, 33418, 33429, 33440, 33442, 33448, 33450, 33812, 33817, 33857, 33860, 33873, 33877,+ 33882, 33889, 33892, 33897, 33940, 33945, 34049, 34057, 34066, 34069, 34074, 34086, 34089, 34112, 34113, 34117,+ 34120, 34129, 34132, 34133, 34134, 34137, 34138, 34149, 34150, 34152, 34154, 34177, 34180, 34182, 34185, 34192,+ 34194, 34197, 34200, 34214, 34321, 34326, 34329, 34341, 34369, 34372, 34377, 34378, 34384, 34389, 34393, 34394,+ 34401, 34406, 34410, 34437, 34449, 34458, 34468, 34816, 34818, 34824, 34826, 34837, 34848, 34850, 34856, 34858,+ 34881, 34885, 34897, 34900, 34905, 34917, 34921, 34944, 34946, 34952, 34954, 34965, 34976, 34978, 34984, 34986,+ 35077, 35078, 35089, 35092, 35094, 35109, 35137, 35140, 35142, 35145, 35152, 35154, 35157, 35162, 35169, 35172,+ 35205, 35222, 35225, 35237, 35328, 35330, 35336, 35338, 35349, 35360, 35362, 35368, 35370, 35397, 35409, 35412,+ 35414, 35456, 35458, 35464, 35466, 35477, 35488, 35490, 35496, 35498, 36869, 36881, 36886, 36888, 36889, 36901,+ 36929, 36934, 36937, 36949, 36952, 36954, 36969, 36970, 36997, 37009, 37012, 37014, 37017, 37029, 37121, 37124,+ 37126, 37129, 37136, 37141, 37144, 37146, 37153, 37156, 37158, 37161, 37184, 37189, 37200, 37201, 37204, 37205,+ 37206, 37209, 37218, 37221, 37252, 37254, 37266, 37269, 37272, 37281, 37284, 37286, 37289, 37381, 37393, 37396,+ 37401, 37413, 37444, 37446, 37449, 37456, 37458, 37461, 37464, 37478, 37481, 37509, 37524, 37526, 37545, 37889,+ 37892, 37894, 37904, 37909, 37912, 37926, 37952, 37962, 37969, 37972, 37973, 37974, 37976, 37977, 37984, 37985,+ 37986, 37989, 38020, 38022, 38034, 38036, 38037, 38040, 38049, 38057, 38144, 38149, 38152, 38154, 38160, 38161,+ 38164, 38165, 38166, 38169, 38177, 38181, 38185, 38186, 38209, 38212, 38213, 38214, 38217, 38224, 38225, 38226,+ 38228, 38229, 38230, 38232, 38233, 38234, 38241, 38244, 38245, 38246, 38249, 38273, 38277, 38280, 38289, 38290,+ 38292, 38293, 38294, 38297, 38298, 38304, 38306, 38309, 38312, 38314, 38401, 38404, 38416, 38421, 38425, 38432,+ 38438, 38441, 38469, 38472, 38473, 38481, 38482, 38485, 38486, 38489, 38501, 38504, 38530, 38532, 38537, 38538,+ 38546, 38548, 38549, 38564, 38566, 38569, 38917, 38934, 38937, 38949, 38977, 38982, 38992, 38994, 38997, 38998,+ 39002, 39012, 39013, 39045, 39057, 39062, 39065, 39077, 39172, 39174, 39177, 39184, 39186, 39189, 39192, 39194,+ 39200, 39201, 39204, 39206, 39232, 39234, 39237, 39240, 39242, 39249, 39252, 39253, 39254, 39257, 39266, 39269,+ 39270, 39274, 39297, 39300, 39312, 39314, 39317, 39322, 39329, 39334, 39429, 39445, 39461, 39492, 39494, 39497,+ 39504, 39509, 39512, 39521, 39557, 39569, 39572, 39573, 39574, 40960, 40962, 40968, 40970, 40981, 40992, 40994,+ 41000, 41002, 41029, 41041, 41044, 41046, 41049, 41088, 41090, 41096, 41098, 41109, 41120, 41122, 41128, 41130,+ 41221, 41225, 41233, 41236, 41238, 41241, 41242, 41286, 41289, 41297, 41301, 41304, 41306, 41313, 41316, 41349,+ 41360, 41362, 41366, 41369, 41474, 41480, 41482, 41488, 41497, 41506, 41512, 41514, 41541, 41553, 41558, 41561,+ 41573, 41600, 41602, 41608, 41610, 41621, 41632, 41634, 41640, 41642, 42009, 42021, 42049, 42052, 42064, 42068,+ 42069, 42072, 42074, 42081, 42085, 42086, 42088, 42089, 42117, 42246, 42249, 42256, 42258, 42261, 42264, 42278,+ 42281, 42306, 42309, 42321, 42324, 42325, 42326, 42329, 42341, 42346, 42369, 42372, 42373, 42374, 42377, 42386,+ 42389, 42392, 42501, 42513, 42518, 42522, 42529, 42533, 42564, 42566, 42570, 42578, 42581, 42582, 42584, 42592,+ 42594, 42630, 42640, 42645, 42646, 42649, 42657, 42660, 42662, 43008, 43010, 43016, 43018, 43040, 43042, 43048,+ 43050, 43089, 43092, 43094, 43097, 43136, 43138, 43144, 43146, 43157, 43168, 43170, 43176, 43178, 43269, 43284,+ 43289, 43297, 43301, 43329, 43344, 43349, 43354, 43361, 43366, 43369, 43408, 43414, 43520, 43522, 43528, 43530,+ 43552, 43554, 43560, 43562, 43601, 43604, 43606, 43648, 43650, 43656, 43658, 43669, 43680, 43682, 43688, 43690,+ };+ static const uint16_t kgrid_2bit_1024[1024] = {+ 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,+ 73, 80, 82, 85, 88, 97, 100, 102, 105, 128, 130, 133, 136, 145, 148, 160,+ 165, 170, 257, 260, 262, 265, 272, 274, 277, 280, 289, 292, 320, 322, 325, 328,+ 337, 340, 342, 345, 352, 357, 360, 385, 388, 400, 402, 405, 417, 420, 512, 514,+ 517, 520, 529, 532, 544, 554, 577, 580, 582, 585, 592, 597, 640, 645, 650, 660,+ 674, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1062, 1065, 1088, 1090, 1093,+ 1096, 1098, 1105, 1108, 1110, 1113, 1120, 1122, 1125, 1153, 1156, 1158, 1161, 1168, 1173, 1176,+ 1185, 1188, 1280, 1282, 1285, 1288, 1290, 1297, 1300, 1302, 1305, 1312, 1317, 1320, 1345, 1348,+ 1350, 1353, 1360, 1362, 1365, 1368, 1377, 1380, 1408, 1410, 1413, 1416, 1425, 1428, 1440, 1537,+ 1540, 1542, 1545, 1552, 1557, 1600, 1605, 1608, 1617, 1620, 1632, 1665, 1668, 1680, 2048, 2050,+ 2053, 2056, 2065, 2068, 2070, 2073, 2080, 2085, 2090, 2113, 2116, 2118, 2121, 2128, 2130, 2133,+ 2136, 2145, 2148, 2176, 2181, 2196, 2218, 2305, 2308, 2320, 2322, 2325, 2328, 2337, 2368, 2373,+ 2376, 2385, 2388, 2400, 2433, 2448, 2560, 2577, 2580, 2594, 2600, 2602, 2640, 2713, 4097, 4100,+ 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4134, 4160, 4162, 4165, 4168, 4177, 4180, 4182,+ 4185, 4192, 4194, 4197, 4200, 4225, 4228, 4230, 4240, 4245, 4248, 4257, 4260, 4352, 4354, 4357,+ 4360, 4362, 4369, 4372, 4374, 4377, 4384, 4386, 4389, 4392, 4417, 4420, 4422, 4425, 4432, 4434,+ 4437, 4440, 4449, 4452, 4480, 4482, 4485, 4488, 4497, 4500, 4609, 4612, 4617, 4624, 4629, 4641,+ 4644, 4672, 4677, 4689, 4692, 4737, 4740, 4752, 5120, 5122, 5125, 5128, 5137, 5140, 5142, 5145,+ 5152, 5157, 5160, 5185, 5188, 5190, 5193, 5200, 5202, 5205, 5208, 5217, 5220, 5248, 5250, 5253,+ 5256, 5265, 5268, 5280, 5377, 5380, 5382, 5385, 5392, 5394, 5397, 5400, 5409, 5412, 5440, 5442,+ 5445, 5448, 5457, 5460, 5472, 5505, 5508, 5520, 5632, 5637, 5640, 5649, 5652, 5664, 5697, 5700,+ 5712, 5760, 5802, 6145, 6148, 6150, 6153, 6160, 6165, 6168, 6177, 6208, 6210, 6213, 6216, 6225,+ 6228, 6240, 6273, 6276, 6400, 6402, 6405, 6408, 6417, 6420, 6432, 6465, 6468, 6480, 6505, 6562,+ 6660, 6672, 6720, 6742, 8192, 8194, 8197, 8200, 8209, 8212, 8214, 8217, 8224, 8229, 8234, 8257,+ 8260, 8272, 8274, 8277, 8292, 8320, 8330, 8340, 8362, 8449, 8452, 8464, 8466, 8469, 8481, 8512,+ 8514, 8517, 8529, 8532, 8544, 8577, 8580, 8592, 8704, 8714, 8738, 8744, 8746, 8772, 8784, 8840,+ 8842, 8872, 9217, 9220, 9222, 9225, 9232, 9237, 9240, 9249, 9252, 9280, 9282, 9285, 9288, 9297,+ 9300, 9312, 9345, 9348, 9360, 9472, 9477, 9480, 9489, 9492, 9504, 9537, 9540, 9552, 9574, 9600,+ 9729, 9732, 9744, 9792, 9817, 10240, 10245, 10257, 10260, 10305, 10308, 10320, 10378, 10410, 10497, 10500,+ 10512, 10645, 10762, 10786, 10852, 10888, 10890, 16385, 16388, 16390, 16393, 16400, 16402, 16405, 16408, 16410,+ 16417, 16420, 16422, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16470, 16473, 16480, 16482, 16485, 16513,+ 16516, 16528, 16533, 16536, 16545, 16548, 16640, 16642, 16645, 16648, 16657, 16660, 16662, 16665, 16672, 16674,+ 16677, 16705, 16708, 16710, 16713, 16720, 16722, 16725, 16728, 16737, 16740, 16768, 16770, 16773, 16776, 16785,+ 16788, 16800, 16897, 16900, 16912, 16914, 16917, 16920, 16932, 16960, 16965, 16968, 16977, 16980, 16992, 17025,+ 17028, 17408, 17410, 17413, 17416, 17418, 17425, 17428, 17430, 17433, 17440, 17442, 17445, 17448, 17473, 17476,+ 17478, 17481, 17488, 17490, 17493, 17496, 17505, 17508, 17536, 17538, 17541, 17544, 17553, 17556, 17568, 17665,+ 17668, 17670, 17673, 17680, 17682, 17685, 17688, 17697, 17700, 17728, 17730, 17733, 17736, 17745, 17748, 17760,+ 17770, 17793, 17796, 17808, 17920, 17922, 17925, 17928, 17937, 17940, 17952, 17985, 17988, 18000, 18048, 18085,+ 18433, 18436, 18441, 18448, 18450, 18453, 18456, 18465, 18468, 18496, 18498, 18501, 18504, 18513, 18516, 18528,+ 18564, 18576, 18688, 18690, 18693, 18696, 18705, 18708, 18720, 18753, 18756, 18768, 18816, 18838, 18945, 18948,+ 18960, 19008, 20480, 20482, 20485, 20488, 20497, 20500, 20502, 20505, 20512, 20514, 20517, 20520, 20545, 20548,+ 20550, 20553, 20560, 20562, 20565, 20568, 20577, 20580, 20608, 20610, 20613, 20616, 20625, 20628, 20737, 20740,+ 20742, 20745, 20752, 20754, 20757, 20760, 20769, 20772, 20800, 20802, 20805, 20808, 20817, 20820, 20832, 20865,+ 20868, 20880, 20992, 20997, 21000, 21009, 21012, 21024, 21057, 21060, 21072, 21097, 21120, 21505, 21508, 21510,+ 21513, 21520, 21522, 21525, 21528, 21537, 21540, 21568, 21570, 21573, 21576, 21585, 21588, 21600, 21633, 21636,+ 21648, 21760, 21762, 21765, 21768, 21777, 21780, 21792, 21825, 21828, 21840, 21888, 22017, 22020, 22032, 22054,+ 22080, 22528, 22530, 22533, 22536, 22545, 22548, 22560, 22593, 22596, 22608, 22618, 22656, 22785, 22788, 22800,+ 22848, 23040, 23065, 23173, 23208, 24577, 24580, 24582, 24592, 24594, 24597, 24600, 24609, 24612, 24640, 24645,+ 24648, 24657, 24660, 24672, 24708, 24720, 24832, 24834, 24837, 24840, 24849, 24852, 24864, 24897, 24900, 24912,+ 24960, 24985, 25092, 25104, 25152, 25174, 25249, 25600, 25605, 25608, 25617, 25620, 25632, 25665, 25668, 25680,+ 25728, 25857, 25860, 25872, 25920, 25930, 25960, 26002, 26112, 26260, 26625, 26628, 26640, 26725, 26776, 26880,+ 26922, 27202, 27297, 32768, 32770, 32773, 32776, 32785, 32788, 32793, 32800, 32805, 32833, 32836, 32848, 32850,+ 32853, 32856, 32865, 32896, 32901, 32913, 32916, 33025, 33028, 33033, 33040, 33042, 33045, 33048, 33057, 33060,+ 33088, 33090, 33093, 33096, 33105, 33108, 33153, 33156, 33168, 33193, 33280, 33285, 33290, 33297, 33300, 33345,+ 33348, 33360, 33793, 33796, 33798, 33801, 33808, 33810, 33813, 33816, 33825, 33856, 33858, 33861, 33864, 33873,+ 33876, 33888, 33921, 33924, 33936, 34048, 34050, 34053, 34056, 34065, 34068, 34080, 34113, 34116, 34128, 34176,+ 34186, 34305, 34308, 34320, 34345, 34368, 34816, 34821, 34833, 34836, 34881, 34884, 34896, 34978, 35073, 35076,+ 35136, 35173, 35362, 35416, 35418, 35458, 35490, 36865, 36868, 36873, 36880, 36882, 36885, 36888, 36900, 36928,+ 36930, 36933, 36936, 36945, 36948, 36960, 36993, 36996, 37008, 37120, 37125, 37137, 37140, 37185, 37188, 37200,+ 37210, 37377, 37380, 37392, 37440, 37542, 37888, 37890, 37893, 37896, 37905, 37908, 37920, 37953, 37956, 37968,+ 38016, 38038, 38145, 38148, 38160, 38208, 38296, 38305, 38400, 38470, 38500, 38913, 38916, 38928, 38950, 38976,+ 39081, 39168, 39241, 39250, 39568, 40960, 40965, 40970, 40980, 40994, 41002, 41025, 41028, 41040, 41122, 41130,+ 41280, 41317, 41474, 41482, 41506, 41512, 41514, 41602, 41608, 41610, 41640, 41985, 41988, 42000, 42048, 42121,+ 42148, 42240, 42265, 42577, 43018, 43048, 43170, 43348, 43398, 43528, 43530, 43552, 43554, 43560, 43656, 43690,};const int kmap_size = 43692;- const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2;+ //const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2;+ const int nwant = type == GGML_TYPE_IQ1_S ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2;const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 :- type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : kgrid_1bit_512;+ type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 :+ type == GGML_TYPE_IQ1_S ? kgrid_1bit_2048 : kgrid_2bit_1024;uint64_t * kgrid_q2xs;int * kmap_q2xs;uint16_t * kneighbors_q2xs;- printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);+ //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t));for (int k = 0; k < grid_size; ++k) {int8_t * pos = (int8_t *)(the_grid + k);@@ -7708,7 +7247,7 @@ void iq2xs_init_impl(enum ggml_type type) {}num_neighbors += n;}- printf("%s: %d neighbours in total\n", __func__, num_neighbors);+ //printf("%s: %d neighbours in total\n", __func__, num_neighbors);kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));iq2_data[gindex].neighbours = kneighbors_q2xs;int counter = 0;@@ -7745,7 +7284,7 @@ void iq2xs_init_impl(enum ggml_type type) {}void iq2xs_free_impl(enum ggml_type type) {- GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S);+ GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ2_S);const int gindex = iq2_data_index(type);if (iq2_data[gindex].grid) {free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL;@@ -7794,7 +7333,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrictconst int kMaxQ = 3;- const int nbl = n/256;+ const int nbl = n/QK_K;block_iq2_xxs * y = vy;@@ -7967,7 +7506,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vconst int kMaxQ = 3;- const int nbl = n/256;+ const int nbl = n/QK_K;block_iq2_xs * y = vy;@@ -8131,8 +7670,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v}}-size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq2_xxs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {GGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;char * qrow = (char *)dst;@@ -8144,8 +7682,7 @@ size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row,return nrow * nblock * sizeof(block_iq2_xxs);}-size_t quantize_iq2_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq2_xs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {GGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;char * qrow = (char *)dst;@@ -8249,7 +7786,7 @@ void iq3xs_init_impl(int grid_size) {int * kmap_q3xs;uint16_t * kneighbors_q3xs;- printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);+ //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);uint32_t * the_grid = (uint32_t *)malloc(grid_size*sizeof(uint32_t));for (int k = 0; k < grid_size; ++k) {int8_t * pos = (int8_t *)(the_grid + k);@@ -8304,7 +7841,7 @@ void iq3xs_init_impl(int grid_size) {}num_neighbors += n;}- printf("%s: %d neighbours in total\n", __func__, num_neighbors);+ //printf("%s: %d neighbours in total\n", __func__, num_neighbors);kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));iq3_data[gindex].neighbours = kneighbors_q3xs;int counter = 0;@@ -8587,8 +8124,7 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v}}-size_t quantize_iq3_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq3_xxs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {GGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;char * qrow = (char *)dst;@@ -8687,7 +8223,8 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo}float best = 0;float scale = max/(2*kMaxQ-1);- for (int is = -15; is <= 15; ++is) {+ for (int k = 0; k < bs4; ++k) is_on_grid[k] = false;+ for (int is = -9; is <= 9; ++is) {float id = (2*kMaxQ-1+is*0.2f)/max;float this_scale = 1/id;for (int k = 0; k < bs4; ++k) {@@ -8723,7 +8260,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, voif (n_not_ongrid > 0 && scale > 0) {float id = 1/scale;for (int k = 0; k < bs4; ++k) {- if (is_on_grid[k]) continue;+ //if (is_on_grid[k]) continue;uint16_t u = 0;for (int i = 0; i < 4; ++i) {int l = nearest_int(0.5f*(id*xval[4*k+i]-1));@@ -8779,7 +8316,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo}float d = max_scale/31;- y[ibl].d = GGML_FP32_TO_FP16(d);+ y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f);float id = 1/d;for (int ib = 0; ib < QK_K/block_size; ib += 2) {int l1 = nearest_int(0.5f*(id*scales[ib+0]-1));@@ -8793,8 +8330,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo}#define IQ3S_BLOCK_SIZE 32-size_t quantize_iq3_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq3_s(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {GGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;float scales[QK_K/IQ3S_BLOCK_SIZE];@@ -8824,7 +8360,7 @@ void quantize_row_iq3_s(const float * restrict x, void * restrict vy, int k) {void quantize_row_iq3_s_reference(const float * restrict x, block_iq3_s * restrict y, int k) {assert(k % QK_K == 0);- quantize_iq3_s(x, y, 1, k, NULL, NULL);+ quantize_iq3_s(x, y, 1, k, NULL);}@@ -8890,12 +8426,70 @@ static int iq1_find_best_neighbour(const uint16_t * restrict neighbours, const ureturn grid_index;}+static int iq1_find_best_neighbour2(const uint16_t * restrict neighbours, const uint64_t * restrict grid,+ const float * restrict xval, const float * restrict weight, float scale, const float * restrict xg, int8_t * restrict L, int ngrid) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_score = FLT_MAX;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float d2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = xg[(pg[i] - 1)/2];+ float w = weight[i];+ float diff = scale*q - xval[i];+ d2 += w*diff*diff;+ }+ if (d2 < best_score) {+ best_score = d2;+ grid_index = neighbours[j];+ }+ }+ if (grid_index < 0) {+ for (int i = 0; i < ngrid; ++i) {+ const int8_t * grid_i = (const int8_t *)(grid + i);+ float d2 = 0;+ for (int j = 0; j < 8; ++j) {+ float w = weight[j];+ float q = xg[(grid_i[j] - 1)/2];+ float diff = scale*q - xval[i];+ d2 += w*diff*diff;+ }+ if (d2 < best_score) {+ best_score = d2;+ grid_index = i;+ }+ }+ }+ if (grid_index < 0) {+ printf("Oops, did not find grid point\n");+ printf("Have %d neighbours\n", num_neighbors);+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = xg[(pg[i] - 1)/2];+ float w = weight[i];+ sumqx += w*q*xval[i];+ sumq2 += w*q*q;+ }+ printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2);+ }+ }+ GGML_ASSERT(grid_index >= 0);+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}+static int iq1_sort_helper(const void * left, const void * right) {const float * l = left;const float * r = right;return *l < *r ? -1 : *l > *r ? 1 : 0;}+#define IQ1S_BLOCK_SIZE 32static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {const int gindex = iq2_data_index(GGML_TYPE_IQ1_S);@@ -8910,41 +8504,45 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vyGGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");GGML_ASSERT(n%QK_K == 0);- const int nbl = n/256;+ const int nbl = n/QK_K;block_iq1_s * y = vy;- float scales[QK_K/8];- float weight[8];- int8_t L[8];- float sumx[9];- float sumw[9];- float pairs[16];+ const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA};+ const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA};++ float scales[QK_K/IQ1S_BLOCK_SIZE];+ float weight[IQ1S_BLOCK_SIZE];+ int8_t L[IQ1S_BLOCK_SIZE];+ float sumx[IQ1S_BLOCK_SIZE+1];+ float sumw[IQ1S_BLOCK_SIZE+1];+ float pairs[2*IQ1S_BLOCK_SIZE];int * idx = (int *)(pairs + 1);- uint8_t hbit[QK_K/8];+ uint16_t index[IQ1S_BLOCK_SIZE/8];+ int8_t shifts[QK_K/IQ1S_BLOCK_SIZE];for (int ibl = 0; ibl < nbl; ++ibl) {y[ibl].d = GGML_FP32_TO_FP16(0.f);memset(y[ibl].qs, 0, QK_K/8);- memset(y[ibl].scales, 0, QK_K/16);+ memset(y[ibl].qh, 0, QK_K/16);float max_scale = 0;const float * xbl = x + QK_K*ibl;float sumx2 = 0;for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];- float sigma2 = sumx2/QK_K;+ float sigma2 = 2*sumx2/QK_K;- for (int ib = 0; ib < QK_K/8; ++ib) {- const float * xb = xbl + 8*ib;- const float * qw = quant_weights + QK_K*ibl + 8*ib;- for (int i = 0; i < 8; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ for (int ib = 0; ib < QK_K/IQ1S_BLOCK_SIZE; ++ib) {+ const float * xb = xbl + IQ1S_BLOCK_SIZE*ib;+ const float * qw = quant_weights + QK_K*ibl + IQ1S_BLOCK_SIZE*ib;+ for (int i = 0; i < IQ1S_BLOCK_SIZE; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);float max = fabsf(xb[0]);- for (int i = 1; i < 8; ++i) max = MAX(max, fabsf(xb[i]));+ for (int i = 1; i < IQ1S_BLOCK_SIZE; ++i) max = MAX(max, fabsf(xb[i]));if (!max) {scales[ib] = 0;- memset(L, 1, 8);+ memset(L, 1, IQ1S_BLOCK_SIZE);continue;}// Here we solve exactly the sum of squared difference (SSD) weighted minimization problem.@@ -8953,52 +8551,81 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy// in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and// Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale// for each possible and score for each split.- for (int j = 0; j < 8; ++j) {+ for (int j = 0; j < IQ1S_BLOCK_SIZE; ++j) {pairs[2*j] = xb[j];idx[2*j] = j;}- qsort(pairs, 8, 2*sizeof(float), iq1_sort_helper);+ qsort(pairs, IQ1S_BLOCK_SIZE, 2*sizeof(float), iq1_sort_helper);{sumx[0] = sumw[0] = 0;- for (int j = 0; j < 8; ++j) {+ for (int j = 0; j < IQ1S_BLOCK_SIZE; ++j) {int i = idx[2*j];sumx[j+1] = sumx[j] + weight[i]*xb[i];sumw[j+1] = sumw[j] + weight[i];}}float best_score = 0, scale = max;- int besti1 = 0, besti2 = 0;- for (int i1 = 0; i1 <= 8; ++i1) {- for (int i2 = i1; i2 <= 8; ++i2) {- float sumqx = -(sumx[i1] - sumx[0]) + (sumx[8] - sumx[i2]);- float sumq2 = (sumw[i1] - sumw[0]) + (sumw[8] - sumw[i2]);+ int besti1 = -1, besti2 = -1, best_shift = 0;+ for (int i1 = 0; i1 <= IQ1S_BLOCK_SIZE; ++i1) {+ for (int i2 = i1; i2 <= IQ1S_BLOCK_SIZE; ++i2) {+ float sumqx = (sumx[i1] - sumx[0])*x_p[0] + (sumx[i2] - sumx[i1])*x_p[1] + (sumx[IQ1S_BLOCK_SIZE] - sumx[i2])*x_p[2];+ float sumq2 = (sumw[i1] - sumw[0])*x_p[0]*x_p[0] + (sumw[i2] - sumw[i1])*x_p[1]*x_p[1] + (sumw[IQ1S_BLOCK_SIZE] - sumw[i2])*x_p[2]*x_p[2];+ if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {+ scale = sumqx/sumq2; best_score = scale*sumqx;+ besti1 = i1; besti2 = i2; best_shift = 1;+ }+ sumqx = (sumx[i1] - sumx[0])*x_m[0] + (sumx[i2] - sumx[i1])*x_m[1] + (sumx[IQ1S_BLOCK_SIZE] - sumx[i2])*x_m[2];+ sumq2 = (sumw[i1] - sumw[0])*x_m[0]*x_m[0] + (sumw[i2] - sumw[i1])*x_m[1]*x_m[1] + (sumw[IQ1S_BLOCK_SIZE] - sumw[i2])*x_m[2]*x_m[2];if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) {scale = sumqx/sumq2; best_score = scale*sumqx;- besti1 = i1; besti2 = i2;+ besti1 = i1; besti2 = i2; best_shift = -1;}}}+ GGML_ASSERT(besti1 >= 0 && besti2 >= 0 && best_shift != 0);for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;- for (int j = besti2; j < 8; ++j) L[idx[2*j]] = 2;+ for (int j = besti2; j < IQ1S_BLOCK_SIZE; ++j) L[idx[2*j]] = 2;if (scale < 0) {- for (int j = 0; j < 8; ++j) L[j] = 2 - L[j];- scale = -scale;+ for (int j = 0; j < IQ1S_BLOCK_SIZE; ++j) L[j] = 2 - L[j];+ scale = -scale; best_shift = -best_shift;+ }+ bool all_on_grid = true;+ const float * xx = best_shift == 1 ? x_p : x_m;+ for (int k = 0; k < IQ1S_BLOCK_SIZE/8; ++k) {+ uint16_t u = 0;+ for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ all_on_grid = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S);+ GGML_ASSERT(grid_index >= 0);+ }+ index[k] = grid_index;+ }+ if (!all_on_grid) {+ float sumqx = 0, sumq2 = 0;+ for (int k = 0; k < IQ1S_BLOCK_SIZE/8; ++k) {+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]);+ for (int j = 0; j < 8; ++j) {+ float w = weight[8*k + j];+ float q = xx[(pg[j] - 1)/2];+ sumqx += w*q*xb[8*k+j];+ sumq2 += w*q*q;+ }+ }+ if (sumqx > 0 && sumq2 > 0) scale = sumqx/sumq2;}- // Now we check if the solution found above corresponds to a grid point and, if not, use a neighbouring- // grid point that minimizes SSD.- uint16_t u = 0;- for (int j = 0; j < 8; ++j) u |= (L[j] << 2*j);- int grid_index = kmap_q2xs[u];- if (grid_index < 0) {- const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;- grid_index = iq1_find_best_neighbour(neighbours, kgrid_q2xs, xb, weight, &scale, L, NGRID_IQ2XXS);- GGML_ASSERT(grid_index >= 0);- }- y[ibl].qs[ib] = grid_index & 255;- hbit[ib] = grid_index >> 8;+ uint16_t h = 0;+ for (int k = 0; k < IQ1S_BLOCK_SIZE/8; ++k) {+ y[ibl].qs[(IQ1S_BLOCK_SIZE/8)*ib + k] = index[k] & 255;+ h |= (index[k] >> 8) << 3*k;+ }+ y[ibl].qh[ib] = h;GGML_ASSERT(scale >= 0);scales[ib] = scale;+ shifts[ib] = best_shift;max_scale = MAX(max_scale, scale);}@@ -9008,19 +8635,18 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy}float d = max_scale/15;- y[ibl].d = GGML_FP32_TO_FP16(d*1.085f); // 1.085f is another fudge factor. Don't ask me why it is needed.+ y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.085f is another fudge factor. Don't ask me why it is needed.float id = 1/d;- for (int ib = 0; ib < QK_K/8; ++ib) {+ for (int ib = 0; ib < QK_K/IQ1S_BLOCK_SIZE; ++ib) {int l = nearest_int(0.5f*(id*scales[ib]-1));l = MAX(0, MIN(7, l));- if (hbit[ib]) l |= 8;- y[ibl].scales[ib/2] |= (l << 4*(ib%2));+ if (shifts[ib] == -1) l |= 8;+ y[ibl].qh[ib] |= (l << 12);}}}-size_t quantize_iq1_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq1_s(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {GGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;char * qrow = (char *)dst;@@ -9045,13 +8671,12 @@ static inline int best_index_int8(int n, const int8_t * val, float x) {return x - val[mu-1] < val[mu] - x ? mu-1 : mu;}-static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * GGML_RESTRICT x,+static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * restrict x,ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l,float * scales, float * weight, uint8_t * L,const int8_t * values,- const float * quant_weights) {-- const int ntry = 7;+ const float * quant_weights,+ const int ntry) {float sigma2 = 0;for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j];@@ -9063,6 +8688,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int blockfloat max_scale = 0, amax_scale = 0;for (int ib = 0; ib < super_block_size/block_size; ++ib) {const float * xb = x + ib*block_size;+ uint8_t * Lb = L + ib*block_size;if (quant_weights) {const float * qw = quant_weights + ib*block_size;for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);@@ -9080,12 +8706,13 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int blockscales[ib] = 0;continue;}- float d = -max/values[0];+ float d = ntry > 0 ? -max/values[0] : max/values[0];float id = 1/d;float sumqx = 0, sumq2 = 0;for (int j = 0; j < block_size; ++j) {float al = id*xb[j];int l = best_index_int8(16, values, al);+ Lb[j] = l;float q = values[l];float w = weight[j];sumqx += w*q*xb[j];@@ -9140,9 +8767,11 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block}} else {dh[0] = GGML_FP32_TO_FP16(scales[0]);- float id = scales[0] ? 1/scales[0] : 0;- for (int j = 0; j < super_block_size; ++j) {- L[j] = best_index_int8(16, values, id*x[j]);+ if (ntry > 0) {+ float id = scales[0] ? 1/scales[0] : 0;+ for (int j = 0; j < super_block_size; ++j) {+ L[j] = best_index_int8(16, values, id*x[j]);+ }}}@@ -9153,8 +8782,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block}}-size_t quantize_iq4_nl(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq4_nl(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {GGML_ASSERT(n_per_row%QK4_NL == 0);int nblock = n_per_row/QK4_NL;char * qrow = (char *)dst;@@ -9168,7 +8796,7 @@ size_t quantize_iq4_nl(const float * src, void * dst, int nrow, int n_per_row, ifor (int ibl = 0; ibl < nblock; ++ibl) {const float * qw = quant_weights ? quant_weights + QK4_NL*ibl : NULL;quantize_row_iq4_nl_impl(QK4_NL, 32, src + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,- &scale, weight, L, kvalues_iq4nl, qw);+ &scale, weight, L, kvalues_iq4nl, qw, 7);}src += n_per_row;qrow += nblock*sizeof(block_iq4_nl);@@ -9177,18 +8805,29 @@ size_t quantize_iq4_nl(const float * src, void * dst, int nrow, int n_per_row, i}void quantize_row_iq4_nl(const float * restrict x, void * restrict vy, int k) {- assert(k % QK4_NL == 0);- block_iq4_nl * restrict y = vy;- quantize_row_iq4_nl_reference(x, y, k);+ GGML_ASSERT(k%QK4_NL == 0);+ int nblock = k/QK4_NL;+ uint8_t L[QK4_NL];+ float weight[QK4_NL];+ uint16_t unused_h;+ uint8_t * unused_l = NULL;+ float scale;+ block_iq4_nl * iq4 = (block_iq4_nl *)vy;+ for (int ibl = 0; ibl < nblock; ++ibl) {+ quantize_row_iq4_nl_impl(QK4_NL, 32, x + QK4_NL*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l,+ &scale, weight, L, kvalues_iq4nl, NULL, -1);+ }}void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * restrict y, int k) {assert(k % QK4_NL == 0);- quantize_iq4_nl(x, y, 1, k, NULL, NULL);+ quantize_row_iq4_nl(x, y, k);}-size_t quantize_iq4_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq4_xs(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {+#if QK_K == 64+ return quantize_iq4_nl(src, dst, nrow, n_per_row, quant_weights);+#elseGGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;char * qrow = (char *)dst;@@ -9200,12 +8839,13 @@ size_t quantize_iq4_xs(const float * src, void * dst, int nrow, int n_per_row, ifor (int ibl = 0; ibl < nblock; ++ibl) {const float * qw = quant_weights ? quant_weights + QK_K*ibl : NULL;quantize_row_iq4_nl_impl(QK_K, 32, src + QK_K*ibl, &iq4[ibl].d, iq4[ibl].qs, &iq4[ibl].scales_h, iq4[ibl].scales_l,- scales, weight, L, kvalues_iq4nl, qw);+ scales, weight, L, kvalues_iq4nl, qw, 7);}src += n_per_row;qrow += nblock*sizeof(block_iq4_xs);}return nrow * nblock * sizeof(block_iq4_xs);+#endif}void quantize_row_iq4_xs(const float * restrict x, void * restrict vy, int k) {@@ -9216,7 +8856,7 @@ void quantize_row_iq4_xs(const float * restrict x, void * restrict vy, int k) {void quantize_row_iq4_xs_reference(const float * restrict x, block_iq4_xs * restrict y, int k) {assert(k % QK_K == 0);- quantize_iq4_xs(x, y, 1, k, NULL, NULL);+ quantize_iq4_xs(x, y, 1, k, NULL);}// =============================== 2.5625 bpw@@ -9236,7 +8876,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vyconst int kMaxQ = 3;- const int nbl = n/256;+ const int nbl = n/QK_K;block_iq2_s * y = vy;@@ -9389,8 +9029,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy}}-size_t quantize_iq2_s(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {- (void)hist;+size_t quantize_iq2_s(const float * restrict src, void * restrict dst, int nrow, int n_per_row, const float * quant_weights) {GGML_ASSERT(n_per_row%QK_K == 0);int nblock = n_per_row/QK_K;char * qrow = (char *)dst;@@ -9404,7 +9043,7 @@ size_t quantize_iq2_s(const float * src, void * dst, int nrow, int n_per_row, invoid quantize_row_iq2_s_reference(const float * restrict x, block_iq2_s * restrict y, int k) {assert(k % QK_K == 0);- quantize_iq2_s(x, y, 1, k, NULL, NULL);+ quantize_iq2_s(x, y, 1, k, NULL);}void quantize_row_iq2_s(const float * restrict x, void * restrict vy, int k) {diff --git a/llama.cpp/ggml-quants.h b/llama.cpp/ggml-quants.hindex 2c61134c..aa7e54a1 100644--- a/llama.cpp/ggml-quants.h+++ b/llama.cpp/ggml-quants.h@@ -1,242 +1,11 @@#pragma once-#include "ggml-impl.h"+#define GGML_COMMON_DECL_C+#include "ggml-common.h"-// GGML internal header--#include <stdint.h>-#include <stddef.h>--#define QK4_0 32-typedef struct {- ggml_fp16_t d; // delta- uint8_t qs[QK4_0 / 2]; // nibbles / quants-} block_q4_0;-static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 block size/padding");--#define QK4_1 32-typedef struct {- ggml_fp16_t d; // delta- ggml_fp16_t m; // min- uint8_t qs[QK4_1 / 2]; // nibbles / quants-} block_q4_1;-static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_fp16_t) + QK4_1 / 2, "wrong q4_1 block size/padding");--#define QK5_0 32-typedef struct {- ggml_fp16_t d; // delta- uint8_t qh[4]; // 5-th bit of quants- uint8_t qs[QK5_0 / 2]; // nibbles / quants-} block_q5_0;-static_assert(sizeof(block_q5_0) == sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");--#define QK5_1 32-typedef struct {- ggml_fp16_t d; // delta- ggml_fp16_t m; // min- uint8_t qh[4]; // 5-th bit of quants- uint8_t qs[QK5_1 / 2]; // nibbles / quants-} block_q5_1;-static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");--#define QK8_0 32-typedef struct {- ggml_fp16_t d; // delta- int8_t qs[QK8_0]; // quants-} block_q8_0;-static_assert(sizeof(block_q8_0) == sizeof(ggml_fp16_t) + QK8_0, "wrong q8_0 block size/padding");--#define QK8_1 32-typedef struct {- float d; // delta- float s; // d * sum(qs[i])- int8_t qs[QK8_1]; // quants-} block_q8_1;-static_assert(sizeof(block_q8_1) == 2*sizeof(float) + QK8_1, "wrong q8_1 block size/padding");--//-// Super-block quantization structures-//+#include "ggml.h"-// Super-block size-#ifdef GGML_QKK_64-#define QK_K 64-#define K_SCALE_SIZE 4-#else-#define QK_K 256-#define K_SCALE_SIZE 12-#endif--// 2-bit quantization-// weight is represented as x = a * q + b-// 16 blocks of 16 elements each-// Effectively 2.625 bits per weight-typedef struct {- uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits- uint8_t qs[QK_K/4]; // quants- ggml_fp16_t d; // super-block scale for quantized scales- ggml_fp16_t dmin; // super-block scale for quantized mins-} block_q2_K;-static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");--// 3-bit quantization-// weight is represented as x = a * q-// 16 blocks of 16 elements each-// Effectively 3.4375 bits per weight-#ifdef GGML_QKK_64-typedef struct {- uint8_t hmask[QK_K/8]; // quants - high bit- uint8_t qs[QK_K/4]; // quants - low 2 bits- uint8_t scales[2];- ggml_fp16_t d; // super-block scale-} block_q3_K;-static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding");-#else-typedef struct {- uint8_t hmask[QK_K/8]; // quants - high bit- uint8_t qs[QK_K/4]; // quants - low 2 bits- uint8_t scales[12]; // scales, quantized with 6 bits- ggml_fp16_t d; // super-block scale-} block_q3_K;-static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");-#endif--// 4-bit quantization-// 8 blocks of 32 elements each-// weight is represented as x = a * q + b-// Effectively 4.5 bits per weight-#ifdef GGML_QKK_64-typedef struct {- ggml_fp16_t d[2]; // super-block scales/mins- uint8_t scales[2]; // 4-bit block scales/mins- uint8_t qs[QK_K/2]; // 4--bit quants-} block_q4_K;-static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + QK_K/2 + 2, "wrong q4_K block size/padding");-#else-typedef struct {- ggml_fp16_t d; // super-block scale for quantized scales- ggml_fp16_t dmin; // super-block scale for quantized mins- uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits- uint8_t qs[QK_K/2]; // 4--bit quants-} block_q4_K;-static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");-#endif--// 5-bit quantization-// 8 blocks of 32 elements each-// weight is represented as x = a * q + b-// Effectively 5.5 bits per weight-#ifdef GGML_QKK_64-typedef struct {- ggml_fp16_t d; // super-block scale- int8_t scales[QK_K/16]; // 8-bit block scales- uint8_t qh[QK_K/8]; // quants, high bit- uint8_t qs[QK_K/2]; // quants, low 4 bits-} block_q5_K;-static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding");-#else-typedef struct {- ggml_fp16_t d; // super-block scale for quantized scales- ggml_fp16_t dmin; // super-block scale for quantized mins- uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits- uint8_t qh[QK_K/8]; // quants, high bit- uint8_t qs[QK_K/2]; // quants, low 4 bits-} block_q5_K;-static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");-#endif--// 6-bit quantization-// weight is represented as x = a * q-// 16 blocks of 16 elements each-// Effectively 6.5625 bits per weight-typedef struct {- uint8_t ql[QK_K/2]; // quants, lower 4 bits- uint8_t qh[QK_K/4]; // quants, upper 2 bits- int8_t scales[QK_K/16]; // scales, quantized with 8 bits- ggml_fp16_t d; // super-block scale-} block_q6_K;-static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");--// This is only used for intermediate quantization and dot products-typedef struct {- float d; // delta- int8_t qs[QK_K]; // quants- int16_t bsums[QK_K/16]; // sum of quants in groups of 16-} block_q8_K;-static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");--// (Almost) "true" 2-bit quantization.-// Due to the need to use blocks as per ggml design, it ends up using-// 2.0625 bpw because of the 16-bit scale for each block of 256.-typedef struct {- ggml_fp16_t d;- uint16_t qs[QK_K/8];-} block_iq2_xxs;-static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");--// 2.3125 bpw quants-typedef struct {- ggml_fp16_t d;- uint16_t qs[QK_K/8];- uint8_t scales[QK_K/32];-} block_iq2_xs;-static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");--// 2.5625 bpw quants-typedef struct {- ggml_fp16_t d;- uint8_t qs[QK_K/4];- uint8_t qh[QK_K/32];- uint8_t scales[QK_K/32];-} block_iq2_s;-static_assert(sizeof(block_iq2_s) == sizeof(ggml_fp16_t) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding");--// (Almost) "true" 3-bit quantization.-// Due to the need to use blocks as per ggml design, it ends up using-// 3.0625 bpw because of the 16-bit scale for each block of 256.-typedef struct {- ggml_fp16_t d;- uint8_t qs[3*QK_K/8];-} block_iq3_xxs;-static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");--// 3.4375 bpw-#if QK_K == 64-#define IQ3S_N_SCALE 2-#else-#define IQ3S_N_SCALE QK_K/64-#endif-typedef struct {- ggml_fp16_t d;- uint8_t qs[QK_K/4];- uint8_t qh[QK_K/32];- uint8_t signs[QK_K/8];- uint8_t scales[IQ3S_N_SCALE];-} block_iq3_s;-static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding");--typedef struct {- ggml_fp16_t d;- uint8_t qs[QK_K/8];- uint8_t scales[QK_K/16];-} block_iq1_s;-static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");--// Non-linear quants-#define QK4_NL 32-typedef struct {- ggml_fp16_t d;- uint8_t qs[QK4_NL/2];-} block_iq4_nl;-static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4_nl block size/padding");--typedef struct {- ggml_fp16_t d;- uint16_t scales_h;- uint8_t scales_l[QK_K/64];- uint8_t qs[QK_K/2];-} block_iq4_xs;-static_assert(sizeof(block_iq4_xs) == sizeof(ggml_fp16_t) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");+// GGML internal header#ifdef __cplusplusextern "C" {@@ -256,6 +25,7 @@ void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGMvoid quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int k);void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int k);void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int k);+void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int k);void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int k);void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int k);@@ -275,6 +45,7 @@ void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, invoid quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);+void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k);@@ -295,6 +66,7 @@ void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRIvoid dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);+void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int k);@@ -316,6 +88,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voivoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);+void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);@@ -325,26 +98,26 @@ void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, constvoid ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);-//// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")-//-size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_iq2_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_iq3_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_iq1_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_iq4_nl (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_iq4_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_iq3_s (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q5_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q6_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q4_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q4_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q5_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);-size_t quantize_q5_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);++size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);+size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int nrows, int n_per_row, const float * imatrix);void iq2xs_init_impl(enum ggml_type type);void iq2xs_free_impl(enum ggml_type type);diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 8bbb0438..57ea4f4c 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -33,7 +33,6 @@ SOFTWARE.");#include "ggml-metal.h"#include "ggml-cuda.h"#include "llamafile/llamafile.h"-#include "llama.cpp/ggml-impl.h"#include "llamafile/log.h"#include <alloca.h>@@ -263,6 +262,17 @@ static ggml_fp16_t ggml_table_exp_f16[1 << 16];// precomputed f32 table for f16 (256 KB) (ggml-impl.h)float ggml_table_f32_f16[1 << 16];+GGML_CALL const char * ggml_status_to_string(enum ggml_status status) {+ switch (status) {+ case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";+ case GGML_STATUS_FAILED: return "GGML status: error (operation failed)";+ case GGML_STATUS_SUCCESS: return "GGML status: success";+ case GGML_STATUS_ABORTED: return "GGML status: warning (operation aborted)";+ }++ return "GGML status: unknown";+}+// note: do not use these inside ggml.c// these are meant to be used via the ggml.h APIfloat ggml_fp16_to_fp32(ggml_fp16_t x) {@@ -312,6 +322,10 @@ void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n) {}}+bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {+ return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;+}+//// timing//@@ -388,6 +402,19 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.type_size = sizeof(int32_t),.is_quantized = false,},+ [GGML_TYPE_I64] = {+ .type_name = "i64",+ .blck_size = 1,+ .type_size = sizeof(int64_t),+ .is_quantized = false,+ },+ [GGML_TYPE_F64] = {+ .type_name = "f64",+ .blck_size = 1,+ .type_size = sizeof(double),+ .is_quantized = false,+ .nrows = 1,+ },[GGML_TYPE_F32] = {.type_name = "f32",.blck_size = 1,@@ -661,14 +688,22 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {},[GGML_TYPE_IQ4_XS] = {.type_name = "iq4_xs",+#if QK_K == 64+ .blck_size = QK4_NL,+#else.blck_size = QK_K,+#endif.type_size = sizeof(block_iq4_xs),.is_quantized = true,.to_float = (ggml_to_float_t) dequantize_row_iq4_xs,.from_float = quantize_row_iq4_xs,.from_float_reference = (ggml_from_float_t)quantize_row_iq4_xs_reference,.vec_dot = ggml_vec_dot_iq4_xs_q8_K,+#if QK_K == 64+ .vec_dot_type = GGML_TYPE_Q8_0,+#else.vec_dot_type = GGML_TYPE_Q8_K,+#endif.nrows = 1,},[GGML_TYPE_Q8_K] = {@@ -787,7 +822,7 @@ do { \#define GGML_F16x8 float16x8_t#define GGML_F16x8_ZERO vdupq_n_f16(0.0f)#define GGML_F16x8_SET1(x) vdupq_n_f16(x)- #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x))+ #define GGML_F16x8_LOAD(x) vld1q_f16((const ggml_fp16_internal_t *)(x))#define GGML_F16x8_STORE vst1q_f16#define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)#define GGML_F16x8_ADD vaddq_f16@@ -830,7 +865,7 @@ do { \#define GGML_F32Cx4 float32x4_t#define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)#define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)- #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x)))+ #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const ggml_fp16_internal_t *)(x)))#define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))#define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)#define GGML_F32Cx4_ADD vaddq_f32@@ -848,6 +883,101 @@ do { \#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE#endif+#elif defined(__AVX512F__)++#define GGML_SIMD++// F32 AVX512++#define GGML_F32_STEP 64+#define GGML_F32_EPR 16++#define GGML_F32x16 __m512+#define GGML_F32x16_ZERO _mm512_setzero_ps()+#define GGML_F32x16_SET1(x) _mm512_set1_ps(x)+#define GGML_F32x16_LOAD _mm512_loadu_ps+#define GGML_F32x16_STORE _mm512_storeu_ps+// _mm512_fmadd_ps is defined in AVX512F so no guard is required+#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)+#define GGML_F32x16_ADD _mm512_add_ps+#define GGML_F32x16_MUL _mm512_mul_ps+#define GGML_F32x16_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ res = _mm512_reduce_add_ps(x[0]); \+} while (0)++// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x16+#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x16_SET1+#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD+#define GGML_F32_VEC_STORE GGML_F32x16_STORE+#define GGML_F32_VEC_FMA GGML_F32x16_FMA+#define GGML_F32_VEC_ADD GGML_F32x16_ADD+#define GGML_F32_VEC_MUL GGML_F32x16_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE++// F16 AVX512++// F16 AVX++#define GGML_F16_STEP 64+#define GGML_F16_EPR 16++// AVX512 has FP16 extension (AVX512_FP16) but I don't have it on my machine so I use FP32 instead++#define GGML_F32Cx16 __m512+#define GGML_F32Cx16_ZERO _mm512_setzero_ps()+#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x)++// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F+// so F16C guard isn't required+#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(x)))+#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))++#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)+#define GGML_F32Cx16_ADD _mm512_add_ps+#define GGML_F32Cx16_MUL _mm512_mul_ps+#define GGML_F32Cx16_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ res = _mm512_reduce_add_ps(x[0]); \+} while (0)++#define GGML_F16_VEC GGML_F32Cx16+#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE+#elif defined(__AVX__)#define GGML_SIMD@@ -1564,9 +1694,15 @@ inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fpinline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {uint16_t t;for (int i = 0; i < n; ++i) {- ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]);- memcpy(&t, &fp16, sizeof(uint16_t));- y[i] = GGML_FP16_TO_FP32(ggml_table_gelu_f16[t]);+ if (x[i] <= -10.0f) {+ y[i] = 0.0f;+ } else if (x[i] >= 10.0f) {+ y[i] = x[i];+ } else {+ ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]);+ memcpy(&t, &fp16, sizeof(uint16_t));+ y[i] = GGML_FP16_TO_FP32(ggml_table_gelu_f16[t]);+ }}}#else@@ -1772,12 +1908,16 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"POOL_2D","UPSCALE","PAD",+ "ARANGE",+ "TIMESTEP_EMBEDDING","ARGSORT","LEAKY_RELU","FLASH_ATTN","FLASH_FF","FLASH_ATTN_BACK",+ "SSM_CONV",+ "SSM_SCAN","WIN_PART","WIN_UNPART","GET_REL_POS",@@ -1800,7 +1940,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"CROSS_ENTROPY_LOSS_BACK",};-static_assert(GGML_OP_COUNT == 72, "GGML_OP_COUNT != 72");+static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"none",@@ -1858,12 +1998,16 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"pool_2d(x)","upscale(x)","pad(x)",+ "arange(start, stop, step)",+ "timestep_embedding(timesteps, dim, max_period)","argsort(x)","leaky_relu(x)","flash_attn(x)","flash_ff(x)","flash_attn_back(x)",+ "ssm_conv(x)",+ "ssm_scan(x)","win_part(x)","win_unpart(x)","get_rel_pos(x)",@@ -1886,7 +2030,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"cross_entropy_loss_back(x,y)",};-static_assert(GGML_OP_COUNT == 72, "GGML_OP_COUNT != 72");+static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");@@ -2465,7 +2609,6 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {GGML_PRINT_DEBUG("%s: g_state initialized in %f ms\n", __func__, (t_end - t_start)/1000.0f);}- ggml_init_cublas();#if defined(GGML_USE_CLBLAST)ggml_cl_init();#elif defined(GGML_USE_VULKAN)@@ -2849,11 +2992,21 @@ static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_return ((const int32_t *)(tensor->op_params))[i];}+static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) {+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));+ return ((const float *)(tensor->op_params))[i];+}+static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));((int32_t *)(tensor->op_params))[i] = value;}+static void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) {+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));+ ((float *)(tensor->op_params))[i] = value;+}+struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor) {memset(tensor->data, 0, ggml_nbytes(tensor));return tensor;@@ -5740,11 +5893,13 @@ struct ggml_tensor * ggml_pool_1d(is_node = true;}- const int64_t ne[2] = {+ const int64_t ne[4] = {ggml_calc_pool_output_size(a->ne[0], k0, s0, p0),a->ne[1],+ a->ne[2],+ a->ne[3],};- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 2, ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);int32_t params[] = { op, k0, s0, p0 };ggml_set_op_params(result, params, sizeof(params));@@ -5850,6 +6005,55 @@ struct ggml_tensor * ggml_upscale(return ggml_upscale_impl(ctx, a, scale_factor);}+struct ggml_tensor * ggml_arange(+ struct ggml_context * ctx,+ float start,+ float stop,+ float step) {++ GGML_ASSERT(stop > start);++ const int64_t steps = (int64_t) ceilf((stop - start) / step);++ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, steps);++ result->op = GGML_OP_ARANGE;+ ggml_set_op_params_f32(result, 0, start);+ ggml_set_op_params_f32(result, 1, stop);+ ggml_set_op_params_f32(result, 2, step);++ return result;+}++struct ggml_tensor * ggml_timestep_embedding(+ struct ggml_context * ctx,+ struct ggml_tensor * timesteps,+ int dim,+ int max_period) {+ bool is_node = false;++ if (timesteps->grad) {+ GGML_ASSERT(false); // TODO: implement backward+ is_node = true;+ }++ int actual_dim = dim;+ if (dim % 2 != 0) {+ actual_dim = dim + 1;+ }++ struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, actual_dim, timesteps->ne[0]);++ result->op = GGML_OP_TIMESTEP_EMBEDDING;+ ggml_set_op_params_i32(result, 0, dim);+ ggml_set_op_params_i32(result, 1, max_period);++ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;+ result->src[0] = timesteps;++ return result;+}+// ggml_argsortstruct ggml_tensor * ggml_argsort(@@ -6029,6 +6233,108 @@ struct ggml_tensor * ggml_flash_attn_back(return result;}+// ggml_ssm_conv++struct ggml_tensor * ggml_ssm_conv(+ struct ggml_context * ctx,+ struct ggml_tensor * s,+ struct ggml_tensor * x,+ struct ggml_tensor * c,+ struct ggml_tensor * sq) {+ GGML_ASSERT(ggml_is_3d(s));+ GGML_ASSERT(ggml_is_matrix(x));+ GGML_ASSERT(ggml_is_matrix(c));+ GGML_ASSERT(ggml_is_matrix(sq));+ GGML_ASSERT(sq->type == GGML_TYPE_I32);++ const int64_t d_conv = c->ne[0];+ const int64_t d_inner = c->ne[1];+ const int64_t n_tokens = x->ne[1];+ const int64_t n_kv = s->ne[2];++ GGML_ASSERT( s->ne[0] == d_conv - 1);+ GGML_ASSERT( s->ne[1] == d_inner);+ GGML_ASSERT( x->ne[0] == d_inner);+ GGML_ASSERT(sq->ne[0] == n_kv);+ GGML_ASSERT(sq->ne[1] == n_tokens);++ bool is_node = false;++ if (s->grad || x->grad || c->grad || sq->grad) {+ GGML_ASSERT(false); // TODO: implement+ is_node = true;+ }++ // 2-in-1 concatenated x and conv_states, {d_inner, n_tokens} with {d_conv, d_inner, n_kv}+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (d_inner*n_tokens) + (d_conv*d_inner*n_kv));++ result->op = GGML_OP_SSM_CONV;+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;+ result->src[0] = s;+ result->src[1] = x;+ result->src[2] = c;+ result->src[3] = sq;++ return result;+}++// ggml_ssm_scan++struct ggml_tensor * ggml_ssm_scan(+ struct ggml_context * ctx,+ struct ggml_tensor * s,+ struct ggml_tensor * x,+ struct ggml_tensor * dt,+ struct ggml_tensor * A,+ struct ggml_tensor * B,+ struct ggml_tensor * C,+ struct ggml_tensor * sq) {+ GGML_ASSERT(ggml_is_contiguous(s));+ GGML_ASSERT(ggml_is_contiguous(x));+ GGML_ASSERT(ggml_is_contiguous(dt));+ GGML_ASSERT(ggml_is_contiguous(A));+ GGML_ASSERT(sq->type == GGML_TYPE_I32);+ GGML_ASSERT(B->nb[0] == ggml_type_size(B->type));+ GGML_ASSERT(C->nb[0] == ggml_type_size(C->type));+ GGML_ASSERT(ggml_are_same_shape(x, dt));++ {+ const int64_t d_state = s->ne[0];+ const int64_t d_inner = s->ne[1];+ const int64_t n_tokens = x->ne[1];++ GGML_ASSERT(x->ne[0] == d_inner);+ GGML_ASSERT(A->ne[0] == d_state);+ GGML_ASSERT(A->ne[1] == d_inner);+ GGML_ASSERT(B->ne[0] == d_state);+ GGML_ASSERT(B->ne[1] == n_tokens);+ GGML_ASSERT(C->ne[0] == d_state);+ GGML_ASSERT(C->ne[1] == n_tokens);+ }++ bool is_node = false;++ if (s->grad || x->grad || dt->grad || A->grad || B->grad || C->grad || sq->grad) {+ GGML_ASSERT(false); // TODO: implement+ is_node = true;+ }++ // 2-in-1 concatenated y and ssm_states, {d_inner, n_tokens} with {d_state, d_inner, n_kv}+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ggml_nelements(x) + ggml_nelements(s));++ result->op = GGML_OP_SSM_SCAN;+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;+ result->src[0] = s;+ result->src[1] = x;+ result->src[2] = dt;+ result->src[3] = A;+ result->src[4] = B;+ result->src[5] = C;+ result->src[6] = sq;++ return result;+}+// ggml_win_partstruct ggml_tensor * ggml_win_part(@@ -10183,7 +10489,7 @@ static void ggml_compute_forward_group_norm_f32(int n_channels = src0->ne[2];int n_groups = dst->op_params[0];int n_channels_per_group = (n_channels + n_groups - 1) / n_groups;- for (int i = ith; i < n_groups; i+=nth) {+ for (int i = ith; i < n_groups; i += nth) {int start = i * n_channels_per_group;int end = start + n_channels_per_group;if (end > n_channels) {@@ -10197,28 +10503,32 @@ static void ggml_compute_forward_group_norm_f32(for (int64_t i01 = 0; i01 < ne01; i01++) {const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);+ ggml_float sumr = 0.0;for (int64_t i00 = 0; i00 < ne00; i00++) {- sum += (ggml_float)x[i00];+ sumr += (ggml_float)x[i00];}+ sum += sumr;}}- float mean = sum / (ne00 * ne01 * step);- ggml_float sum2 = 0.0;+ const float mean = sum / (ne00 * ne01 * step);+ ggml_float sum2 = 0.0;for (int64_t i02 = start; i02 < end; i02++) {for (int64_t i01 = 0; i01 < ne01; i01++) {const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);+ ggml_float sumr = 0.0;for (int64_t i00 = 0; i00 < ne00; i00++) {float v = x[i00] - mean;y[i00] = v;- sum2 += (ggml_float)(v * v);+ sumr += (ggml_float)(v * v);}+ sum2 += sumr;}}- float variance = sum2 / (ne00 * ne01 * step);+ const float variance = sum2 / (ne00 * ne01 * step);const float scale = 1.0f / sqrtf(variance + eps);for (int64_t i02 = start; i02 < end; i02++) {@@ -10786,7 +11096,6 @@ static void ggml_compute_forward_out_prod_f32(// nb01 >= nb00 - src0 is not transposed// compute by src0 rows- // TODO: #if defined(GGML_USE_CUBLAS) ggml_cuda_out_prod// TODO: #if defined(GGML_USE_CLBLAST)#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)@@ -10986,7 +11295,6 @@ static void ggml_compute_forward_out_prod_q_f32(// nb01 >= nb00 - src0 is not transposed// compute by src0 rows- // TODO: #if defined(GGML_USE_CUBLAS) ggml_cuda_out_prod// TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST)if (params->type == GGML_TASK_TYPE_INIT) {@@ -11349,8 +11657,6 @@ static void ggml_compute_forward_get_rows_q(const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- assert(params->ith == 0);-if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {return;}@@ -11358,7 +11664,7 @@ static void ggml_compute_forward_get_rows_q(GGML_TENSOR_BINARY_OP_LOCALSconst int64_t nc = ne00;- const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr);+ const int64_t nr = ggml_nelements(src1);const enum ggml_type type = src0->type;ggml_to_float_t const dequantize_row_q = type_traits[type].to_float;@@ -11368,17 +11674,25 @@ static void ggml_compute_forward_get_rows_q(assert(nb00 == ggml_type_size(type));assert(ggml_nrows(dst) == nr);- // TODO: multi-thread- for (int64_t i12 = 0; i12 < ne12; ++i12) {- for (int64_t i11 = 0; i11 < ne11; ++i11) {- for (int64_t i10 = 0; i10 < ne10; ++i10) {- const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);+ const int ith = params->ith;+ const int nth = params->nth;++ // rows per thread+ const int dr = (nr + nth - 1)/nth;- dequantize_row_q(- (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),- (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);- }- }+ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int64_t i = ir0; i < ir1; ++i) {+ const int64_t i12 = i/(ne11*ne10);+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);++ dequantize_row_q(+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);}}@@ -11389,8 +11703,6 @@ static void ggml_compute_forward_get_rows_f16(const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- assert(params->ith == 0);-if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {return;}@@ -11398,24 +11710,32 @@ static void ggml_compute_forward_get_rows_f16(GGML_TENSOR_BINARY_OP_LOCALSconst int64_t nc = ne00;- const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr);+ const int64_t nr = ggml_nelements(src1);assert(ne0 == nc);assert(ne02 == ne11);assert(nb00 == sizeof(ggml_fp16_t));assert(ggml_nrows(dst) == nr);- // TODO: multi-thread- for (int64_t i12 = 0; i12 < ne12; ++i12) {- for (int64_t i11 = 0; i11 < ne11; ++i11) {- for (int64_t i10 = 0; i10 < ne10; ++i10) {- const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);+ const int ith = params->ith;+ const int nth = params->nth;- ggml_fp16_to_fp32_row(- (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),- (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);- }- }+ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int64_t i = ir0; i < ir1; ++i) {+ const int64_t i12 = i/(ne11*ne10);+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);++ ggml_fp16_to_fp32_row(+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);}}@@ -11426,8 +11746,6 @@ static void ggml_compute_forward_get_rows_f32(const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- assert(params->ith == 0);-if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {return;}@@ -11435,24 +11753,32 @@ static void ggml_compute_forward_get_rows_f32(GGML_TENSOR_BINARY_OP_LOCALSconst int64_t nc = ne00;- const int64_t nr = ggml_nelements(src1); GGML_UNUSED(nr);+ const int64_t nr = ggml_nelements(src1);assert(ne0 == nc);assert(ne02 == ne11);assert(nb00 == sizeof(float));assert(ggml_nrows(dst) == nr);- // TODO: multi-thread- for (int64_t i12 = 0; i12 < ne12; ++i12) {- for (int64_t i11 = 0; i11 < ne11; ++i11) {- for (int64_t i10 = 0; i10 < ne10; ++i10) {- const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);+ const int ith = params->ith;+ const int nth = params->nth;- ggml_vec_cpy_f32(nc,- (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3),- (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03));- }- }+ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int64_t i = ir0; i < ir1; ++i) {+ const int64_t i12 = i/(ne11*ne10);+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);++ ggml_vec_cpy_f32(nc,+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3),+ (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03));}}@@ -12189,6 +12515,8 @@ static void ggml_compute_forward_alibi(case GGML_TYPE_I8:case GGML_TYPE_I16:case GGML_TYPE_I32:+ case GGML_TYPE_I64:+ case GGML_TYPE_F64:case GGML_TYPE_COUNT:{GGML_ASSERT(false);@@ -12275,6 +12603,8 @@ static void ggml_compute_forward_clamp(case GGML_TYPE_I8:case GGML_TYPE_I16:case GGML_TYPE_I32:+ case GGML_TYPE_I64:+ case GGML_TYPE_F64:case GGML_TYPE_COUNT:{GGML_ASSERT(false);@@ -13523,6 +13853,106 @@ static void ggml_compute_forward_pad(}}++// ggml_compute_forward_arange++static void ggml_compute_forward_arange_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ GGML_ASSERT(dst->nb[0] == sizeof(float));++ const int ith = params->ith;+ const int nth = params->nth;++ const float start = ggml_get_op_params_f32(dst, 0);+ const float stop = ggml_get_op_params_f32(dst, 1);+ const float step = ggml_get_op_params_f32(dst, 2);++ const int64_t steps = (int64_t) ceilf((stop - start) / step);++ GGML_ASSERT(ggml_nelements(dst) == steps);++ for (int64_t i = ith; i < steps; i+= nth) {+ float value = start + step * i;+ ((float *)dst->data)[i] = value;+ }+}++static void ggml_compute_forward_arange(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {+ switch (dst->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_arange_f32(params, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}++static void ggml_compute_forward_timestep_embedding_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const struct ggml_tensor * src0 = dst->src[0];++ GGML_ASSERT(src0->nb[0] == sizeof(float));++ const int ith = params->ith;+ const int nth = params->nth;++ GGML_TENSOR_UNARY_OP_LOCALS++ const int dim = ggml_get_op_params_i32(dst, 0);+ const int max_period = ggml_get_op_params_i32(dst, 1);++ int half = dim / 2;++ for (int64_t i = 0; i < ne00; i++) {+ float * embed_data = (float *)((char *) dst->data + i*nb1);+ for (int64_t j = ith; j < half; j += nth) {+ float timestep = ((float *)src0->data)[i];+ float freq = (float)expf(-logf(max_period) * j / half);+ float arg = timestep * freq;+ embed_data[j] = cosf(arg);+ embed_data[j + half] = sinf(arg);+ }+ if (dim % 2 != 0 && ith == 0) {+ embed_data[dim] = 0.f;+ }+ }+}++static void ggml_compute_forward_timestep_embedding(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ switch (src0->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_timestep_embedding_f32(params, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}+// ggml_compute_forward_argsortstatic void ggml_compute_forward_argsort_f32(@@ -14566,6 +14996,257 @@ static void ggml_compute_forward_flash_attn_back(}}+// ggml_compute_forward_ssm_conv++static void ggml_compute_forward_ssm_conv_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const struct ggml_tensor * src0 = dst->src[0]; // conv_state+ const struct ggml_tensor * src1 = dst->src[1]; // x+ const struct ggml_tensor * src2 = dst->src[2]; // conv1d.weight+ const struct ggml_tensor * src3 = dst->src[3]; // state_seq++ const int ith = params->ith;+ const int nth = params->nth;++ const int nc = src2->ne[0]; // d_conv+ const int nr = src0->ne[1]; // d_inner+ const int n_t = src1->ne[1]; // n_tokens+ const int n_kv = src0->ne[2]; // max number of sequences in the batch++ GGML_ASSERT((nr*n_t) + (nc*nr*n_kv) == ggml_nelements(dst));+ GGML_ASSERT(src0->nb[0] == sizeof(float));+ GGML_ASSERT(src1->nb[0] == sizeof(float));+ GGML_ASSERT(src2->nb[0] == sizeof(float));+ GGML_ASSERT(src3->nb[0] == sizeof(int32_t));+ GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float));+ // for use with the destination state offset between sequences+ GGML_ASSERT(src2->nb[2] == src2->ne[1]*src2->ne[0]*sizeof(float));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);+ const int ir = ir1 - ir0;++ if (n_kv > 1) {+ // multiple sequences means it's hard to know when it's the first time a state is read,+ // so copy them all over to the destination, just to be sure.+ for (int i3 = 0; i3 < n_kv; ++i3) {+ float * s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + i3*(src0->nb[2]));+ float * s = (float *) ((char *) dst->data + ir0*(src2->nb[1]) + i3*(src2->nb[2]) + nr*n_t*sizeof(float));+ // can't use memcpy because of d_conv vs d_conv - 1+ for (int i1 = 0; i1 < ir; ++i1) {+ for (int i0 = 0; i0 < nc - 1; ++i0) {+ // copy s0 to last (d_conv - 1) columns of s+ s[1 + i0 + i1*nc] = s0[i0 + i1*(nc - 1)];+ }+ }+ }+ }++ for (int i2 = 0; i2 < n_t; ++i2) {+ int32_t * sq = (int32_t *) ((char *) src3->data + i2*(src3->nb[1])); // {n_kv, n_tokens}+ float * x = (float *) ((char *) dst->data + ir0*sizeof(float) + i2*(nr*sizeof(float))); // {d_inner, n_tokens}+ float * s = (float *) ((char *) dst->data + ir0*(src2->nb[1]) + sq[0]*(src2->nb[2]) + nr*n_t*sizeof(float)); // {d_conv, d_inner, n_kv}+ float * s0; // {d_conv - 1, d_inner, n_kv}+ float * x0 = (float *) ((char *) src1->data + ir0*(src1->nb[0]) + i2*(src1->nb[1])); // {d_inner, n_tokens}+ float * c = (float *) ((char *) src2->data + ir0*(src2->nb[1])); // {d_conv, d_inner}+ int ne0s0;++ GGML_ASSERT(0 <= sq[0] && sq[0] < n_kv);++ // avoid needing to copy the state for the first token+ if (i2 == 0) {+ s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + sq[0]*(src0->nb[2])); // {d_conv - 1, d_inner, n_kv}+ ne0s0 = src0->ne[0];+ } else {+ // the source is the last (d_conv - 1) columns of the destination+ s0 = s + 1;+ ne0s0 = nc;+ }++ // d_inner+ for (int i1 = 0; i1 < ir; ++i1) {+ // shift state left+ for (int i0 = 0; i0 < nc - 1; ++i0) {+ s[i0 + i1*nc] = s0[i0 + i1*ne0s0];+ }+ // insert x on the last column+ s[(nc - 1) + i1*nc] = x0[i1];+ }++ // handle copies when there are multiple output states+ for (int i3 = 1; i3 < n_kv; ++i3) {+ int32_t seq = sq[i3];+ if (0 <= seq && seq < n_kv) {+ float * s1 = s + (seq - sq[0])*nc*nr;+ memcpy(s1, s, nc*ir*sizeof(float));+ } else {+ // stop at negative or too big seq_ids+ break;+ }+ }++ // it seems a little faster when this is separate from the state shift+ for (int i1 = 0; i1 < ir; ++i1) {+ // rowwise dot product+ float sumf = 0.0f;+ for (int i0 = 0; i0 < nc; ++i0) {+ int i = i0 + i1*nc;+ sumf += s[i] * c[i];+ }+ x[i1] = sumf;+ }+ }+}++static void ggml_compute_forward_ssm_conv(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {+ switch (dst->src[0]->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_ssm_conv_f32(params, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}++// ggml_compute_forward_ssm_scan++static void ggml_compute_forward_ssm_scan_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const struct ggml_tensor * src0 = dst->src[0]; // s+ const struct ggml_tensor * src1 = dst->src[1]; // x+ const struct ggml_tensor * src2 = dst->src[2]; // dt+ const struct ggml_tensor * src3 = dst->src[3]; // A+ const struct ggml_tensor * src4 = dst->src[4]; // B+ const struct ggml_tensor * src5 = dst->src[5]; // C+ const struct ggml_tensor * src6 = dst->src[6]; // sq++ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t nc = src0->ne[0]; // d_state+ const int64_t nr = src0->ne[1]; // d_inner+ const int64_t n_t = src1->ne[1]; // number of tokens in the batch+ const int64_t n_kv = src0->ne[2]; // max number of sequences in the batch++ GGML_ASSERT(ggml_nelements(src1) + ggml_nelements(src0) == ggml_nelements(dst));+ GGML_ASSERT(src0->nb[0] == sizeof(float));+ GGML_ASSERT(src1->nb[0] == sizeof(float));+ GGML_ASSERT(src2->nb[0] == sizeof(float));+ GGML_ASSERT(src3->nb[0] == sizeof(float));+ GGML_ASSERT(src4->nb[0] == sizeof(float));+ GGML_ASSERT(src5->nb[0] == sizeof(float));+ // required for the dot product between s and C, and when copying the states+ GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float));+ // required for per-sequence offsets for states+ GGML_ASSERT(src0->nb[2] == src0->ne[0]*src0->ne[1]*sizeof(float));+ // required to get correct offset for state destination (i.e. src1->nb[2])+ GGML_ASSERT(src1->nb[2] == src1->ne[0]*src1->ne[1]*sizeof(float));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);+ const int ir = ir1 - ir0;++ if (n_kv > 1) {+ // it's hard to know if the source states have already been copied+ // when there are multiple, so copy them already.+ for (int i3 = 0; i3 < n_kv; ++i3) {+ float * s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + i3*(src0->nb[2]));+ float * s = (float *) ((char *) dst->data + ir0*(src0->nb[1]) + i3*(src0->nb[2]) + src1->nb[2]);+ memcpy(s, s0, nc*ir*sizeof(float));+ }+ }++ for (int i2 = 0; i2 < n_t; ++i2) {+ int32_t * sq = (int32_t *) ((char *) src6->data + i2*(src6->nb[1])); // {n_kv, n_tokens}+ float * y = (float *) ((char *) dst->data + ir0*(src1->nb[0]) + i2*(src1->nb[1])); // {d_inner, n_tokens}+ float * s = (float *) ((char *) dst->data + ir0*(src0->nb[1]) + sq[0]*(src0->nb[2]) + src1->nb[2]); // {d_state, d_inner, n_kv}+ float * s0;+ float * x = (float *) ((char *) src1->data + ir0*(src1->nb[0]) + i2*(src1->nb[1])); // {d_inner, n_tokens}+ float * dt = (float *) ((char *) src2->data + ir0*(src2->nb[0]) + i2*(src2->nb[1])); // {d_inner, n_tokens}+ float * A = (float *) ((char *) src3->data + ir0*(src3->nb[1])); // {d_state, d_inner}+ float * B = (float *) ((char *) src4->data + i2*(src4->nb[1])); // {d_state, n_tokens}+ float * C = (float *) ((char *) src5->data + i2*(src5->nb[1])); // {d_state, n_tokens}++ GGML_ASSERT(0 <= sq[0] && sq[0] < n_kv);++ // avoid needing to copy the state for the first token+ if (i2 == 0) {+ s0 = (float *) ((char *) src0->data + ir0*(src0->nb[1]) + sq[0]*(src0->nb[2])); // {d_state, d_inner, n_kv}+ } else {+ // otherwise the source is the same as the destination+ s0 = s;+ }++ // d_inner+ for (int i1 = 0; i1 < ir; ++i1) {+ // ref: https://github.com/state-spaces/mamba/blob/34076d664838588a3c97727b263478ab9f621a07/mamba_ssm/ops/triton/selective_state_update.py#L78+ float dt_soft_plus = dt[i1] <= 20.0f ? log1pf(expf(dt[i1])) : dt[i1];+ float x_dt = x[i1] * dt_soft_plus;+ float sumf = 0.0f;+ // d_state+ for (int i0 = 0; i0 < nc; ++i0) {+ int i = i0 + i1*nc;+ // state = prev_state * dA + dB * x+ float state = (s0[i] * expf(dt_soft_plus * A[i])) + (B[i0] * x_dt);+ // y = rowwise_dotprod(state, C)+ sumf += state * C[i0];+ s[i] = state;+ }+ y[i1] = sumf;+ }++ // handle copies when there are multiple output states+ for (int i3 = 1; i3 < n_kv; ++i3) {+ int32_t seq = sq[i3];+ if (0 <= seq && seq < n_kv) {+ float * s1 = s + (seq - sq[0])*nc*nr;+ memcpy(s1, s, nc*ir*sizeof(float));+ } else {+ // stop at negative or too big seq_ids+ break;+ }+ }+ }+}++static void ggml_compute_forward_ssm_scan(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {+ switch (dst->src[0]->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_ssm_scan_f32(params, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}+// ggml_compute_forward_win_partstatic void ggml_compute_forward_win_part_f32(@@ -15065,9 +15746,10 @@ static void ggml_compute_forward_map_custom1(return;}- struct ggml_map_custom1_op_params * p = (struct ggml_map_custom1_op_params *) dst->op_params;+ struct ggml_map_custom1_op_params p;+ memcpy(&p, dst->op_params, sizeof(p));- p->fun(dst, a, params->ith, params->nth, p->userdata);+ p.fun(dst, a, params->ith, params->nth, p.userdata);}// ggml_compute_forward_map_custom2@@ -15083,9 +15765,10 @@ static void ggml_compute_forward_map_custom2(return;}- struct ggml_map_custom2_op_params * p = (struct ggml_map_custom2_op_params *) dst->op_params;+ struct ggml_map_custom2_op_params p;+ memcpy(&p, dst->op_params, sizeof(p));- p->fun(dst, a, b, params->ith, params->nth, p->userdata);+ p.fun(dst, a, b, params->ith, params->nth, p.userdata);}// ggml_compute_forward_map_custom3@@ -15102,9 +15785,10 @@ static void ggml_compute_forward_map_custom3(return;}- struct ggml_map_custom3_op_params * p = (struct ggml_map_custom3_op_params *) dst->op_params;+ struct ggml_map_custom3_op_params p;+ memcpy(&p, dst->op_params, sizeof(p));- p->fun(dst, a, b, c, params->ith, params->nth, p->userdata);+ p.fun(dst, a, b, c, params->ith, params->nth, p.userdata);}// ggml_compute_forward_cross_entropy_loss@@ -15356,28 +16040,19 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggmreturn;}-if (ggml_cublas_loaded()) {-/* #ifdef GGML_USE_CUBLAS */- bool skip_cpu = ggml_cuda_compute_forward(params, tensor);+#if defined(GGML_USE_VULKAN)+ const bool skip_cpu = ggml_vk_compute_forward_cpu_assist(params, tensor);+#ifdef GGML_VULKAN_CHECK_RESULTS+ if (skip_cpu) {+ ggml_vk_check_results_1_cpu_assist(params, tensor);+ }+#endifif (skip_cpu) {return;}GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_TYPE_CPU);GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_TYPE_CPU);-/* #elif defined(GGML_USE_VULKAN) */-/* const bool skip_cpu = ggml_vk_compute_forward_cpu_assist(params, tensor); */-/* #ifdef GGML_VULKAN_CHECK_RESULTS */-/* if (skip_cpu) { */-/* ggml_vk_check_results_1_cpu_assist(params, tensor); */-/* } */-/* #endif */-/* if (skip_cpu) { */-/* return; */-/* } */-/* GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU); */-/* GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU); */-/* #endif // GGML_USE_CUBLAS */-}+#endif // GGML_USE_VULKAN#ifdef GGML_USE_SYCLbool skip_cpu = ggml_sycl_compute_forward(params, tensor);@@ -15590,6 +16265,14 @@ if (ggml_cublas_loaded()) {{ggml_compute_forward_pad(params, tensor);} break;+ case GGML_OP_ARANGE:+ {+ ggml_compute_forward_arange(params, tensor);+ } break;+ case GGML_OP_TIMESTEP_EMBEDDING:+ {+ ggml_compute_forward_timestep_embedding(params, tensor);+ } break;case GGML_OP_ARGSORT:{ggml_compute_forward_argsort(params, tensor);@@ -15616,6 +16299,14 @@ if (ggml_cublas_loaded()) {bool masked = t != 0;ggml_compute_forward_flash_attn_back(params, masked, tensor);} break;+ case GGML_OP_SSM_CONV:+ {+ ggml_compute_forward_ssm_conv(params, tensor);+ } break;+ case GGML_OP_SSM_SCAN:+ {+ ggml_compute_forward_ssm_scan(params, tensor);+ } break;case GGML_OP_WIN_PART:{ggml_compute_forward_win_part(params, tensor);@@ -16592,6 +17283,14 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor{GGML_ASSERT(false); // TODO: not implemented} break;+ case GGML_OP_ARANGE:+ {+ GGML_ASSERT(false); // TODO: not implemented+ } break;+ case GGML_OP_TIMESTEP_EMBEDDING:+ {+ GGML_ASSERT(false); // TODO: not implemented+ } break;case GGML_OP_ARGSORT:{GGML_ASSERT(false); // TODO: not implemented@@ -16662,6 +17361,11 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor{GGML_ASSERT(false); // not supported} break;+ case GGML_OP_SSM_CONV:+ case GGML_OP_SSM_SCAN:+ {+ GGML_ASSERT(false); // TODO: not implemented+ } break;case GGML_OP_WIN_PART:case GGML_OP_WIN_UNPART:case GGML_OP_UNARY:@@ -17192,6 +17896,7 @@ struct ggml_compute_state {ggml_thread_t thrd;int ith;struct ggml_compute_state_shared * shared;+ enum ggml_status ec;};static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const struct ggml_compute_state_shared * st) {@@ -17203,7 +17908,7 @@ static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, constnode->perf_time_us += time_us_cur;}-static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {+static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_threads) {int n_tasks = 0;switch (node->op) {@@ -17284,6 +17989,12 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {{n_tasks = n_threads;} break;+ case GGML_OP_GET_ROWS:+ {+ // FIXME: the cost of launching additional threads decreases performance with GPU offloading+ //n_tasks = MIN(n_threads, ggml_nelements(node->src[1]));+ n_tasks = MIN(n_cur_threads, ggml_nelements(node->src[1]));+ } break;case GGML_OP_SCALE:case GGML_OP_SET:case GGML_OP_CONT:@@ -17291,7 +18002,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {case GGML_OP_VIEW:case GGML_OP_PERMUTE:case GGML_OP_TRANSPOSE:- case GGML_OP_GET_ROWS:case GGML_OP_GET_ROWS_BACK:case GGML_OP_DIAG:{@@ -17343,6 +18053,14 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {{n_tasks = n_threads;} break;+ case GGML_OP_ARANGE:+ {+ n_tasks = n_threads;+ } break;+ case GGML_OP_TIMESTEP_EMBEDDING:+ {+ n_tasks = n_threads;+ } break;case GGML_OP_ARGSORT:{n_tasks = n_threads;@@ -17359,6 +18077,11 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {{n_tasks = n_threads;} break;+ case GGML_OP_SSM_CONV:+ case GGML_OP_SSM_SCAN:+ {+ n_tasks = n_threads;+ } break;case GGML_OP_WIN_PART:case GGML_OP_WIN_UNPART:case GGML_OP_GET_REL_POS:@@ -17372,29 +18095,32 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {} break;case GGML_OP_MAP_CUSTOM1:{- struct ggml_map_custom1_op_params * p = (struct ggml_map_custom1_op_params *) node->op_params;- if (p->n_tasks == GGML_N_TASKS_MAX) {+ struct ggml_map_custom1_op_params p;+ memcpy(&p, node->op_params, sizeof(p));+ if (p.n_tasks == GGML_N_TASKS_MAX) {n_tasks = n_threads;} else {- n_tasks = MIN(p->n_tasks, n_threads);+ n_tasks = MIN(p.n_tasks, n_threads);}} break;case GGML_OP_MAP_CUSTOM2:{- struct ggml_map_custom2_op_params * p = (struct ggml_map_custom2_op_params *) node->op_params;- if (p->n_tasks == GGML_N_TASKS_MAX) {+ struct ggml_map_custom2_op_params p;+ memcpy(&p, node->op_params, sizeof(p));+ if (p.n_tasks == GGML_N_TASKS_MAX) {n_tasks = n_threads;} else {- n_tasks = MIN(p->n_tasks, n_threads);+ n_tasks = MIN(p.n_tasks, n_threads);}} break;case GGML_OP_MAP_CUSTOM3:{- struct ggml_map_custom3_op_params * p = (struct ggml_map_custom3_op_params *) node->op_params;- if (p->n_tasks == GGML_N_TASKS_MAX) {+ struct ggml_map_custom3_op_params p;+ memcpy(&p, node->op_params, sizeof(p));+ if (p.n_tasks == GGML_N_TASKS_MAX) {n_tasks = n_threads;} else {- n_tasks = MIN(p->n_tasks, n_threads);+ n_tasks = MIN(p.n_tasks, n_threads);}} break;case GGML_OP_CROSS_ENTROPY_LOSS:@@ -17474,7 +18200,8 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {while (true) {if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {state->shared->node_n += 1;- return (thread_ret_t) GGML_EXIT_ABORTED;+ state->ec = GGML_STATUS_ABORTED;+ return 0;}if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {@@ -17492,7 +18219,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {/* FINALIZE */struct ggml_tensor * node = cgraph->nodes[node_n];if (GGML_OP_HAS_FINALIZE[node->op]) {- params.nth = ggml_get_n_tasks(node, n_threads);+ params.nth = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);ggml_compute_forward(¶ms, node);}ggml_graph_compute_perf_stats_node(node, state->shared);@@ -17502,7 +18229,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {while (++node_n < cgraph->n_nodes) {GGML_PRINT_DEBUG_5("%s: %d/%d\n", __func__, node_n, cgraph->n_nodes);struct ggml_tensor * node = cgraph->nodes[node_n];- const int n_tasks = ggml_get_n_tasks(node, n_threads);+ const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);state->shared->perf_node_start_cycles = ggml_perf_cycles();state->shared->perf_node_start_time_us = ggml_perf_time_us();@@ -17550,7 +18277,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {/* INIT & COMPUTE */struct ggml_tensor * node = cgraph->nodes[node_n];- const int n_tasks = ggml_get_n_tasks(node, n_threads);+ const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);struct ggml_compute_params params = {/*.type =*/ GGML_TASK_TYPE_INIT,@@ -17596,7 +18323,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {}}- return GGML_EXIT_SUCCESS;+ return 0;}struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threads) {@@ -17615,7 +18342,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threafor (int i = 0; i < cgraph->n_nodes; i++) {struct ggml_tensor * node = cgraph->nodes[i];- const int n_tasks = ggml_get_n_tasks(node, n_threads);+ const int n_tasks = ggml_get_n_tasks(node, n_threads, 1);max_tasks = MAX(max_tasks, n_tasks);@@ -17792,7 +18519,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threareturn cplan;}-int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {+enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {{GGML_ASSERT(cplan);GGML_ASSERT(cplan->n_threads > 0);@@ -17836,6 +18563,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {.thrd = 0,.ith = j,.shared = &state_shared,+ .ec = GGML_STATUS_SUCCESS,};const int rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);@@ -17846,12 +18574,14 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {workers[0].ith = 0;workers[0].shared = &state_shared;+ workers[0].ec = GGML_STATUS_SUCCESS;const int64_t perf_start_cycles = ggml_perf_cycles();const int64_t perf_start_time_us = ggml_perf_time_us();// this is a work thread too- int compute_status = (size_t) ggml_graph_compute_thread(&workers[0]);+ ggml_graph_compute_thread(&workers[0]);+ enum ggml_status compute_status = workers[0].ec;// don't leave affinity set on the main threadclear_numa_thread_affinity();@@ -17861,6 +18591,8 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {for (int j = 1; j < n_threads; j++) {const int rc = ggml_thread_join(workers[j].thrd, NULL);GGML_ASSERT(rc == 0);+ if (workers[j].ec != GGML_STATUS_SUCCESS)+ compute_status = workers[j].ec;}}@@ -17888,14 +18620,14 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {return compute_status;}-void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {+enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads);struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, cplan.work_size);cplan.work_data = (uint8_t *)ctx->mem_buffer + obj->offs;- ggml_graph_compute(cgraph, &cplan);+ return ggml_graph_compute(cgraph, &cplan);}struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name) {@@ -18002,7 +18734,7 @@ void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) {// write binary data{- FILE * fout = fopen(fname, "wbe");+ FILE * fout = fopen(fname, "wb");if (!fout) {fprintf(stderr, "%s: failed to open %s\n", __func__, fname);@@ -18140,7 +18872,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *// read file into data{- FILE * fin = fopen(fname, "rbe");+ FILE * fin = fopen(fname, "rb");if (!fin) {fprintf(stderr, "%s: failed to open %s\n", __func__, fname);return result;@@ -18476,7 +19208,7 @@ static void ggml_graph_dump_dot_leaf_edge(FILE * fp, struct ggml_tensor * node,void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename) {char color[16];- FILE * fp = fopen(filename, "we");+ FILE * fp = fopen(filename, "w");GGML_ASSERT(fp);fprintf(fp, "digraph G {\n");@@ -19544,133 +20276,6 @@ void ggml_quantize_free(void) {ggml_critical_section_end();}-size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist) {- assert(k % QK4_0 == 0);- const int nb = k / QK4_0;-- for (int b = 0; b < n; b += k) {- block_q4_0 * restrict y = (block_q4_0 *) dst + b/QK4_0;-- quantize_row_q4_0_reference(src + b, y, k);-- for (int i = 0; i < nb; i++) {- for (int j = 0; j < QK4_0; j += 2) {- const uint8_t vi0 = y[i].qs[j/2] & 0x0F;- const uint8_t vi1 = y[i].qs[j/2] >> 4;-- hist[vi0]++;- hist[vi1]++;- }- }- }-- return (n/QK4_0*sizeof(block_q4_0));-}--size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist) {- assert(k % QK4_1 == 0);- const int nb = k / QK4_1;-- for (int b = 0; b < n; b += k) {- block_q4_1 * restrict y = (block_q4_1 *) dst + b/QK4_1;-- quantize_row_q4_1_reference(src + b, y, k);-- for (int i = 0; i < nb; i++) {- for (int j = 0; j < QK4_1; j += 2) {- const uint8_t vi0 = y[i].qs[j/2] & 0x0F;- const uint8_t vi1 = y[i].qs[j/2] >> 4;-- hist[vi0]++;- hist[vi1]++;- }- }- }-- return (n/QK4_1*sizeof(block_q4_1));-}--size_t ggml_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t * hist) {- assert(k % QK5_0 == 0);- const int nb = k / QK5_0;-- for (int b = 0; b < n; b += k) {- block_q5_0 * restrict y = (block_q5_0 *)dst + b/QK5_0;-- quantize_row_q5_0_reference(src + b, y, k);-- for (int i = 0; i < nb; i++) {- uint32_t qh;- memcpy(&qh, &y[i].qh, sizeof(qh));-- for (int j = 0; j < QK5_0; j += 2) {- const uint8_t vh0 = ((qh & (1u << (j/2 + 0 ))) >> (j/2 + 0 )) << 4;- const uint8_t vh1 = ((qh & (1u << (j/2 + 16))) >> (j/2 + 12));-- // cast to 16 bins- const uint8_t vi0 = ((y[i].qs[j/2] & 0x0F) | vh0) / 2;- const uint8_t vi1 = ((y[i].qs[j/2] >> 4) | vh1) / 2;-- hist[vi0]++;- hist[vi1]++;- }- }- }-- return (n/QK5_0*sizeof(block_q5_0));-}--size_t ggml_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t * hist) {- assert(k % QK5_1 == 0);- const int nb = k / QK5_1;-- for (int b = 0; b < n; b += k) {- block_q5_1 * restrict y = (block_q5_1 *)dst + b/QK5_1;-- quantize_row_q5_1_reference(src + b, y, k);-- for (int i = 0; i < nb; i++) {- uint32_t qh;- memcpy(&qh, &y[i].qh, sizeof(qh));-- for (int j = 0; j < QK5_1; j += 2) {- const uint8_t vh0 = ((qh & (1u << (j/2 + 0 ))) >> (j/2 + 0 )) << 4;- const uint8_t vh1 = ((qh & (1u << (j/2 + 16))) >> (j/2 + 12));-- // cast to 16 bins- const uint8_t vi0 = ((y[i].qs[j/2] & 0x0F) | vh0) / 2;- const uint8_t vi1 = ((y[i].qs[j/2] >> 4) | vh1) / 2;-- hist[vi0]++;- hist[vi1]++;- }- }- }-- return (n/QK5_1*sizeof(block_q5_1));-}--size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist) {- assert(k % QK8_0 == 0);- const int nb = k / QK8_0;-- for (int b = 0; b < n; b += k) {- block_q8_0 * restrict y = (block_q8_0 *)dst + b/QK8_0;-- quantize_row_q8_0_reference(src + b, y, k);-- for (int i = 0; i < nb; i++) {- for (int j = 0; j < QK8_0; ++j) {- const int8_t vi = y[i].qs[j];-- hist[vi/16 + 8]++;- }- }- }-- return (n/QK8_0*sizeof(block_q8_0));-}-bool ggml_quantize_requires_imatrix(enum ggml_type type) {returntype == GGML_TYPE_IQ2_XXS ||@@ -19678,173 +20283,53 @@ bool ggml_quantize_requires_imatrix(enum ggml_type type) {type == GGML_TYPE_IQ1_S;}-size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start,- int nrows, int n_per_row, int64_t * hist, const float * imatrix) {+size_t ggml_quantize_chunk(+ enum ggml_type type,+ const float * src,+ void * dst,+ int start,+ int nrows,+ int n_per_row,+ const float * imatrix) {+ const int n = nrows * n_per_row;++ if (ggml_quantize_requires_imatrix(type)) {+ GGML_ASSERT(imatrix != NULL);+ }++ GGML_ASSERT(start % type_traits[type].blck_size == 0);+ GGML_ASSERT(start % n_per_row == 0);+ggml_quantize_init(type); // this is noop if already initialized++ const size_t start_row = start / n_per_row;+ const size_t row_size = ggml_row_size(type, n_per_row);+size_t result = 0;- int n = nrows * n_per_row;+switch (type) {- case GGML_TYPE_Q4_0:- {- GGML_ASSERT(start % QK4_0 == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q4_0(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q4_1:- {- GGML_ASSERT(start % QK4_1 == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q4_1(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q5_0:- {- GGML_ASSERT(start % QK5_0 == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q5_0(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q5_1:- {- GGML_ASSERT(start % QK5_1 == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q5_1(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q8_0:- {- GGML_ASSERT(start % QK8_0 == 0);- block_q8_0 * block = (block_q8_0*)dst + start / QK8_0;- result = ggml_quantize_q8_0(src + start, block, n, n, hist);- } break;- case GGML_TYPE_Q2_K:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q2_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q3_K:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q3_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q4_K:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q4_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q5_K:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q5_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_Q6_K:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_q6_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ2_XXS:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- GGML_ASSERT(imatrix);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq2_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ2_XS:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- GGML_ASSERT(imatrix);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq2_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ3_XXS:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq3_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ3_S:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq3_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ2_S:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq2_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ1_S:- {- GGML_ASSERT(start % QK_K == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq1_s(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ4_NL:- {- GGML_ASSERT(start % QK4_NL == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq4_nl(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;- case GGML_TYPE_IQ4_XS:- {- GGML_ASSERT(start % QK4_NL == 0);- GGML_ASSERT(start % n_per_row == 0);- size_t start_row = start / n_per_row;- size_t row_size = ggml_row_size(type, n_per_row);- result = quantize_iq4_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);- GGML_ASSERT(result == row_size * nrows);- } break;+ case GGML_TYPE_Q4_0: result = quantize_q4_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q4_1: result = quantize_q4_1(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q5_0: result = quantize_q5_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q5_1: result = quantize_q5_1(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q8_0: result = quantize_q8_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q2_K: result = quantize_q2_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q3_K: result = quantize_q3_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q4_K: result = quantize_q4_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q5_K: result = quantize_q5_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_Q6_K: result = quantize_q6_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ2_XS: result = quantize_iq2_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ3_XXS: result = quantize_iq3_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ3_S: result = quantize_iq3_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ2_S: result = quantize_iq2_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ1_S: result = quantize_iq1_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+ case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+#if QK_K == 64+ case GGML_TYPE_IQ4_XS: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+#else+ case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;+#endifcase GGML_TYPE_F16:{size_t elemsize = sizeof(ggml_fp16_t);@@ -19860,6 +20345,9 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, idefault:assert(false);}++ GGML_ASSERT(result == nrows * row_size);+return result;}@@ -21087,7 +21575,7 @@ int ggml_cpu_has_blas(void) {}int ggml_cpu_has_cublas(void) {- return ggml_cublas_loaded();+ return llamafile_has_cuda();}int ggml_cpu_has_clblast(void) {diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 8116e41e..8977645f 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -318,6 +318,16 @@extern "C" {#endif+ enum ggml_status {+ GGML_STATUS_ALLOC_FAILED = -2,+ GGML_STATUS_FAILED = -1,+ GGML_STATUS_SUCCESS = 0,+ GGML_STATUS_ABORTED = 1,+ };++ // get ggml_status name string+ GGML_API GGML_CALL const char * ggml_status_to_string(enum ggml_status status);+typedef uint16_t ggml_fp16_t;// convert FP16 <-> FP32@@ -330,24 +340,24 @@ extern "C" {struct ggml_object;struct ggml_context;+ // NOTE: always add types at the end of the enum to keep backward compatibilityenum ggml_type {- GGML_TYPE_F32 = 0,- GGML_TYPE_F16 = 1,- GGML_TYPE_Q4_0 = 2,- GGML_TYPE_Q4_1 = 3,+ GGML_TYPE_F32 = 0,+ GGML_TYPE_F16 = 1,+ GGML_TYPE_Q4_0 = 2,+ GGML_TYPE_Q4_1 = 3,// GGML_TYPE_Q4_2 = 4, support has been removed- // GGML_TYPE_Q4_3 (5) support has been removed- GGML_TYPE_Q5_0 = 6,- GGML_TYPE_Q5_1 = 7,- GGML_TYPE_Q8_0 = 8,- GGML_TYPE_Q8_1 = 9,- // k-quantizations- GGML_TYPE_Q2_K = 10,- GGML_TYPE_Q3_K = 11,- GGML_TYPE_Q4_K = 12,- GGML_TYPE_Q5_K = 13,- GGML_TYPE_Q6_K = 14,- GGML_TYPE_Q8_K = 15,+ // GGML_TYPE_Q4_3 = 5, support has been removed+ GGML_TYPE_Q5_0 = 6,+ GGML_TYPE_Q5_1 = 7,+ GGML_TYPE_Q8_0 = 8,+ GGML_TYPE_Q8_1 = 9,+ GGML_TYPE_Q2_K = 10,+ GGML_TYPE_Q3_K = 11,+ GGML_TYPE_Q4_K = 12,+ GGML_TYPE_Q5_K = 13,+ GGML_TYPE_Q6_K = 14,+ GGML_TYPE_Q8_K = 15,GGML_TYPE_IQ2_XXS = 16,GGML_TYPE_IQ2_XS = 17,GGML_TYPE_IQ3_XXS = 18,@@ -356,9 +366,11 @@ extern "C" {GGML_TYPE_IQ3_S = 21,GGML_TYPE_IQ2_S = 22,GGML_TYPE_IQ4_XS = 23,- GGML_TYPE_I8,- GGML_TYPE_I16,- GGML_TYPE_I32,+ GGML_TYPE_I8 = 24,+ GGML_TYPE_I16 = 25,+ GGML_TYPE_I32 = 26,+ GGML_TYPE_I64 = 27,+ GGML_TYPE_F64 = 28,GGML_TYPE_COUNT,};@@ -376,20 +388,20 @@ extern "C" {// model file typesenum ggml_ftype {- GGML_FTYPE_UNKNOWN = -1,- GGML_FTYPE_ALL_F32 = 0,- GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors- GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors- GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors+ GGML_FTYPE_UNKNOWN = -1,+ GGML_FTYPE_ALL_F32 = 0,+ GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensorsGGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16- GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors- GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors- GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors- GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors- GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors- GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors- GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors- GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors@@ -457,12 +469,16 @@ extern "C" {GGML_OP_POOL_2D,GGML_OP_UPSCALE, // nearest interpolateGGML_OP_PAD,+ GGML_OP_ARANGE,+ GGML_OP_TIMESTEP_EMBEDDING,GGML_OP_ARGSORT,GGML_OP_LEAKY_RELU,GGML_OP_FLASH_ATTN,GGML_OP_FLASH_FF,GGML_OP_FLASH_ATTN_BACK,+ GGML_OP_SSM_CONV,+ GGML_OP_SSM_SCAN,GGML_OP_WIN_PART,GGML_OP_WIN_UNPART,GGML_OP_GET_REL_POS,@@ -675,6 +691,16 @@ extern "C" {GGML_NUMA_STRATEGY_COUNT};+ //+ // GUID+ //++ // GUID types+ typedef uint8_t ggml_guid[16];+ typedef ggml_guid * ggml_guid_t;++ GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);+// miscGGML_API void ggml_time_init(void); // call this once at the beginning of the program@@ -1654,6 +1680,15 @@ extern "C" {int p2,int p3);+ // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151+ // timesteps: [N,]+ // return: [N, dim]+ GGML_API struct ggml_tensor * ggml_timestep_embedding(+ struct ggml_context * ctx,+ struct ggml_tensor * timesteps,+ int dim,+ int max_period);+// sort rowsenum ggml_sort_order {GGML_SORT_ORDER_ASC,@@ -1665,6 +1700,12 @@ extern "C" {struct ggml_tensor * a,enum ggml_sort_order order);+ GGML_API struct ggml_tensor * ggml_arange(+ struct ggml_context * ctx,+ float start,+ float stop,+ float step);+// top k elements per rowGGML_API struct ggml_tensor * ggml_top_k(struct ggml_context * ctx,@@ -1694,6 +1735,23 @@ extern "C" {struct ggml_tensor * c0,struct ggml_tensor * c1);+ GGML_API struct ggml_tensor * ggml_ssm_conv(+ struct ggml_context * ctx,+ struct ggml_tensor * s,+ struct ggml_tensor * x,+ struct ggml_tensor * c,+ struct ggml_tensor * sq);++ GGML_API struct ggml_tensor * ggml_ssm_scan(+ struct ggml_context * ctx,+ struct ggml_tensor * s,+ struct ggml_tensor * x,+ struct ggml_tensor * dt,+ struct ggml_tensor * A,+ struct ggml_tensor * B,+ struct ggml_tensor * C,+ struct ggml_tensor * sq);+// partition into non-overlapping windows with padding if needed// example:// a: 768 64 64 1@@ -1916,12 +1974,11 @@ extern "C" {// ggml_graph_plan() has to be called before ggml_graph_compute()// when plan.work_size > 0, caller must allocate memory for plan.work_data- GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);- GGML_API int ggml_graph_compute( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);-+ GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);+ GGML_API enum ggml_status ggml_graph_compute ( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);// same as ggml_graph_compute() but the work data is allocated as a part of the context// note: the drawback of this API is that you must have ensured that the context has enough memory for the work data- GGML_API void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);+ GGML_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name);@@ -2142,25 +2199,18 @@ extern "C" {GGML_API void ggml_quantize_init(enum ggml_type type);GGML_API void ggml_quantize_free(void);- // TODO: these would probably get removed in favor of the more general ggml_quantize_chunk- GGML_API size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist);-- GGML_API size_t ggml_quantize_q2_K(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q3_K(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist);-// some quantization type cannot be used without an importance matrixGGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);// calls ggml_quantize_init internally (i.e. can allocate memory)- GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst,- int start, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+ GGML_API size_t ggml_quantize_chunk(+ enum ggml_type type,+ const float * src,+ void * dst,+ int start,+ int nrows,+ int n_per_row,+ const float * imatrix);//// ggufdiff --git a/llama.cpp/grammar-parser.cpp b/llama.cpp/grammar-parser.cppindex bf89a96f..2a130156 100644--- a/llama.cpp/grammar-parser.cpp+++ b/llama.cpp/grammar-parser.cpp@@ -278,6 +278,22 @@ namespace grammar_parser {while (*pos) {pos = parse_rule(state, pos);}+ // Validate the state to ensure that all rules are defined+ for (const auto & rule : state.rules) {+ for (const auto & elem : rule) {+ if (elem.type == LLAMA_GRETYPE_RULE_REF) {+ // Ensure that the rule at that location exists+ if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) {+ // Get the name of the rule that is missing+ for (const auto & kv : state.symbol_ids) {+ if (kv.second == elem.value) {+ throw std::runtime_error("Undefined rule identifier '" + kv.first + "'");+ }+ }+ }+ }+ }+ }return state;} catch (const std::exception & err) {fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what());diff --git a/llama.cpp/imatrix/imatrix.cpp b/llama.cpp/imatrix/imatrix.cppindex 088db919..cdf31f03 100644--- a/llama.cpp/imatrix/imatrix.cpp+++ b/llama.cpp/imatrix/imatrix.cpp@@ -60,13 +60,31 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void *const struct ggml_tensor * src0 = t->src[0];const struct ggml_tensor * src1 = t->src[1];+ std::string wname;+ {+ // remove any prefix and suffixes from the name+ // CUDA0#blk.0.attn_k.weight#0 => blk.0.attn_k.weight+ const char * p = strchr(src0->name, '#');+ if (p != NULL) {+ p = p + 1;+ const char * q = strchr(p, '#');+ if (q != NULL) {+ wname = std::string(p, q - p);+ } else {+ wname = p;+ }+ } else {+ wname = src0->name;+ }+ }+// when ask is true, the scheduler wants to know if we are interested in data from this tensor// if we return true, a follow-up call will be made with ask=false in which we can do the actual collectionif (ask) {if (t->op == GGML_OP_MUL_MAT_ID) return true; // collect all indirect matrix multiplicationsif (t->op != GGML_OP_MUL_MAT) return false;if (src1->ne[1] < 16 || src1->type != GGML_TYPE_F32) return false;- if (!(strncmp(src0->name, "blk.", 4) == 0 || (m_params.collect_output_weight && strcmp(src0->name, "output.weight") == 0))) return false;+ if (!(wname.substr(0, 4) == "blk." || (m_params.collect_output_weight && wname == "output.weight"))) return false;return true;}@@ -98,12 +116,12 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void *// this is necessary to guarantee equal number of "ncall" for each tensorfor (int ex = 0; ex < n_as; ++ex) {src0 = t->src[2 + ex];- auto& e = m_stats[src0->name];+ auto& e = m_stats[wname];if (e.values.empty()) {e.values.resize(src1->ne[0], 0);}else if (e.values.size() != (size_t)src1->ne[0]) {- fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", src0->name, (int)e.values.size(), (int)src1->ne[0]);+ fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", wname.c_str(), (int)e.values.size(), (int)src1->ne[0]);exit(1); //GGML_ASSERT(false);}// NOTE: since we select top-k experts, the number of calls for the expert tensors will be k times larger@@ -111,7 +129,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void *//if (idx == t->src[0]->ne[0] - 1) ++e.ncall;++e.ncall;if (m_params.verbosity > 1) {- printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, src0->name, ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type);+ printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, wname.c_str(), ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type);}for (int row = 0; row < (int)src1->ne[1]; ++row) {const int excur = m_ids[row*n_as + idx];@@ -133,17 +151,17 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void *}}} else {- auto& e = m_stats[src0->name];+ auto& e = m_stats[wname];if (e.values.empty()) {e.values.resize(src1->ne[0], 0);}else if (e.values.size() != (size_t)src1->ne[0]) {- fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", src0->name, (int)e.values.size(), (int)src1->ne[0]);+ fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", wname.c_str(), (int)e.values.size(), (int)src1->ne[0]);exit(1); //GGML_ASSERT(false);}++e.ncall;if (m_params.verbosity > 1) {- printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, src0->name, ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type);+ printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, wname.c_str(), ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type);}for (int row = 0; row < (int)src1->ne[1]; ++row) {const float * x = data + row * src1->ne[0];diff --git a/llama.cpp/json-schema-to-grammar.cpp b/llama.cpp/json-schema-to-grammar.cppnew file mode 100644index 00000000..dc6d30ef--- /dev/null+++ b/llama.cpp/json-schema-to-grammar.cpp@@ -0,0 +1,725 @@+#include "json-schema-to-grammar.h"+#include <algorithm>+#include <fstream>+#include <map>+#include <regex>+#include <sstream>+#include <string>+#include <unordered_map>+#include <unordered_set>+#include <vector>++using json = nlohmann::json;++const std::string SPACE_RULE = "\" \"?";++std::unordered_map<std::string, std::string> PRIMITIVE_RULES = {+ {"boolean", "(\"true\" | \"false\") space"},+ {"number", "(\"-\"? ([0-9] | [1-9] [0-9]*)) (\".\" [0-9]+)? ([eE] [-+]? [0-9]+)? space"},+ {"integer", "(\"-\"? ([0-9] | [1-9] [0-9]*)) space"},+ {"value", "object | array | string | number | boolean"},+ {"object", "\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space"},+ {"array", "\"[\" space ( value (\",\" space value)* )? \"]\" space"},+ {"uuid", "\"\\\"\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "+ "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "+ "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "+ "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "+ "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] \"\\\"\" space"},+ {"string", " \"\\\"\" (\n"+ " [^\"\\\\] |\n"+ " \"\\\\\" ([\"\\\\/bfnrt] | \"u\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])\n"+ " )* \"\\\"\" space"},+ {"null", "\"null\" space"}+};+std::vector<std::string> OBJECT_RULE_NAMES = {"object", "array", "string", "number", "boolean", "null", "value"};++std::unordered_map<std::string, std::string> DATE_RULES = {+ {"date", "[0-9] [0-9] [0-9] [0-9] \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )"},+ {"time", "([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9] [0-9] [0-9] )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )"},+ {"date-time", "date \"T\" time"},+ {"date-string", "\"\\\"\" date \"\\\"\" space"},+ {"time-string", "\"\\\"\" time \"\\\"\" space"},+ {"date-time-string", "\"\\\"\" date-time \"\\\"\" space"}+};++static bool is_reserved_name(const std::string & name) {+ static std::unordered_set<std::string> RESERVED_NAMES;+ if (RESERVED_NAMES.empty()) {+ RESERVED_NAMES.insert("root");+ for (const auto &p : PRIMITIVE_RULES) RESERVED_NAMES.insert(p.first);+ for (const auto &p : DATE_RULES) RESERVED_NAMES.insert(p.first);+ }+ return RESERVED_NAMES.find(name) != RESERVED_NAMES.end();+}++std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+");+std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"]");+std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]");+std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {+ {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}+};++std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'};+std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};++template <typename Iterator>+std::string join(Iterator begin, Iterator end, const std::string & separator) {+ std::ostringstream result;+ if (begin != end) {+ result << *begin;+ for (Iterator it = begin + 1; it != end; ++it) {+ result << separator << *it;+ }+ }+ return result.str();+}++static std::vector<std::string> split(const std::string & str, const std::string & delimiter) {+ std::vector<std::string> tokens;+ size_t start = 0;+ size_t end = str.find(delimiter);++ while (end != std::string::npos) {+ tokens.push_back(str.substr(start, end - start));+ start = end + delimiter.length();+ end = str.find(delimiter, start);+ }++ tokens.push_back(str.substr(start));++ return tokens;+}++static std::string repeat(const std::string & str, size_t n) {+ if (n == 0) {+ return "";+ }++ std::string result;+ result.reserve(str.length() * n);++ for (size_t i = 0; i < n; ++i) {+ result += str;+ }++ return result;+}++static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function<std::string(const std::smatch &)> & replacement) {+ std::smatch match;+ std::string result;++ std::string::const_iterator searchStart(input.cbegin());+ std::string::const_iterator searchEnd(input.cend());++ while (std::regex_search(searchStart, searchEnd, match, regex)) {+ result.append(searchStart, searchStart + match.position());+ result.append(replacement(match));+ searchStart = match.suffix().first;+ }++ result.append(searchStart, searchEnd);++ return result;+}++static std::string format_literal(const std::string & literal) {+ std::string escaped = replacePattern(json(literal).dump(), GRAMMAR_LITERAL_ESCAPE_RE, [&](const std::smatch & match) {+ char c = match.str()[0];+ return GRAMMAR_LITERAL_ESCAPES.at(c);+ });+ return "\"" + escaped + "\"";+}+++class SchemaConverter {+private:+ std::function<json(const std::string &)> _fetch_json;+ bool _dotall;+ std::map<std::string, std::string> _rules;+ std::unordered_map<std::string, nlohmann::json> _refs;+ std::unordered_set<std::string> _refs_being_resolved;+ std::vector<std::string> _errors;+ std::vector<std::string> _warnings;++ std::string _add_rule(const std::string & name, const std::string & rule) {+ std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");+ if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {+ _rules[esc_name] = rule;+ return esc_name;+ } else {+ int i = 0;+ while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) {+ i++;+ }+ std::string key = esc_name + std::to_string(i);+ _rules[key] = rule;+ return key;+ }+ }++ std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {+ std::vector<std::string> rules;+ for (size_t i = 0; i < alt_schemas.size(); i++) {+ rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));+ }+ return join(rules.begin(), rules.end(), " | ");+ }++ std::string _visit_pattern(const std::string & pattern, const std::string & name) {+ if (!(pattern.front() == '^' && pattern.back() == '$')) {+ _errors.push_back("Pattern must start with '^' and end with '$'");+ return "";+ }+ std::string sub_pattern = pattern.substr(1, pattern.length() - 2);+ std::unordered_map<std::string, std::string> sub_rule_ids;++ size_t i = 0;+ size_t length = sub_pattern.length();++ using literal_or_rule = std::pair<std::string, bool>;+ auto to_rule = [&](const literal_or_rule & ls) {+ auto is_literal = ls.second;+ auto s = ls.first;+ return is_literal ? "\"" + s + "\"" : s;+ };+ std::function<literal_or_rule()> transform = [&]() -> literal_or_rule {+ size_t start = i;+ std::vector<literal_or_rule> seq;++ auto get_dot = [&]() {+ std::string rule;+ if (_dotall) {+ rule = "[\\U00000000-\\U0010FFFF]";+ } else {+ rule = "[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]";+ }+ return _add_rule("dot", rule);+ };++ // Joins the sequence, merging consecutive literals together.+ auto join_seq = [&]() {+ std::vector<literal_or_rule> ret;++ std::string literal;+ auto flush_literal = [&]() {+ if (literal.empty()) {+ return false;+ }+ ret.push_back(std::make_pair(literal, true));+ literal.clear();+ return true;+ };++ for (const auto & item : seq) {+ auto is_literal = item.second;+ if (is_literal) {+ literal += item.first;+ } else {+ flush_literal();+ ret.push_back(item);+ }+ }+ flush_literal();++ std::vector<std::string> results;+ for (const auto & item : ret) {+ results.push_back(to_rule(item));+ }+ return std::make_pair(join(results.begin(), results.end(), " "), false);+ };++ while (i < length) {+ char c = sub_pattern[i];+ if (c == '.') {+ seq.push_back(std::make_pair(get_dot(), false));+ i++;+ } else if (c == '(') {+ i++;+ if (i < length) {+ if (sub_pattern[i] == '?') {+ _warnings.push_back("Unsupported pattern syntax");+ }+ }+ seq.push_back(std::make_pair("(" + to_rule(transform()) + ")", false));+ } else if (c == ')') {+ i++;+ if (start > 0 && sub_pattern[start - 1] != '(') {+ _errors.push_back("Unbalanced parentheses");+ }+ return join_seq();+ } else if (c == '[') {+ std::string square_brackets = std::string(1, c);+ i++;+ while (i < length && sub_pattern[i] != ']') {+ if (sub_pattern[i] == '\\') {+ square_brackets += sub_pattern.substr(i, 2);+ i += 2;+ } else {+ square_brackets += sub_pattern[i];+ i++;+ }+ }+ if (i >= length) {+ _errors.push_back("Unbalanced square brackets");+ }+ square_brackets += ']';+ i++;+ seq.push_back(std::make_pair(square_brackets, false));+ } else if (c == '|') {+ seq.push_back(std::make_pair("|", false));+ i++;+ } else if (c == '*' || c == '+' || c == '?') {+ seq.back() = std::make_pair(to_rule(seq.back()) + c, false);+ i++;+ } else if (c == '{') {+ std::string curly_brackets = std::string(1, c);+ i++;+ while (i < length && sub_pattern[i] != '}') {+ curly_brackets += sub_pattern[i];+ i++;+ }+ if (i >= length) {+ _errors.push_back("Unbalanced curly brackets");+ }+ curly_brackets += '}';+ i++;+ auto nums = split(curly_brackets.substr(1, curly_brackets.length() - 2), ",");+ int min_times = 0;+ int max_times = std::numeric_limits<int>::max();+ try {+ if (nums.size() == 1) {+ min_times = max_times = std::stoi(nums[0]);+ } else if (nums.size() != 2) {+ _errors.push_back("Wrong number of values in curly brackets");+ } else {+ if (!nums[0].empty()) {+ min_times = std::stoi(nums[0]);+ }+ if (!nums[1].empty()) {+ max_times = std::stoi(nums[1]);+ }+ }+ } catch (const std::invalid_argument & e) {+ _errors.push_back("Invalid number in curly brackets");+ return std::make_pair("", false);+ }+ auto &last = seq.back();+ auto &sub = last.first;+ auto sub_is_literal = last.second;++ if (min_times == 0 && max_times == std::numeric_limits<int>::max()) {+ sub += "*";+ } else if (min_times == 0 && max_times == 1) {+ sub += "?";+ } else if (min_times == 1 && max_times == std::numeric_limits<int>::max()) {+ sub += "+";+ } else {+ if (!sub_is_literal) {+ std::string & sub_id = sub_rule_ids[sub];+ if (sub_id.empty()) {+ sub_id = _add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub);+ }+ sub = sub_id;+ }+ std::string result;+ if (sub_is_literal && min_times > 0) {+ result = "\"" + repeat(sub.substr(1, sub.length() - 2), min_times) + "\"";+ } else {+ for (int j = 0; j < min_times; j++) {+ if (j > 0) {+ result += " ";+ }+ result += sub;+ }+ }+ if (min_times > 0 && min_times < max_times) {+ result += " ";+ }+ if (max_times == std::numeric_limits<int>::max()) {+ result += sub + "*";+ } else {+ for (int j = min_times; j < max_times; j++) {+ if (j > min_times) {+ result += " ";+ }+ result += sub + "?";+ }+ }+ seq.back().first = result;+ seq.back().second = false;+ }+ } else {+ std::string literal;+ auto is_non_literal = [&](char c) {+ return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end();+ };+ while (i < length) {+ if (sub_pattern[i] == '\\' && i < length - 1) {+ char next = sub_pattern[i + 1];+ if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) {+ i++;+ literal += sub_pattern[i];+ i++;+ } else {+ literal += sub_pattern.substr(i, 2);+ i += 2;+ }+ } else if (sub_pattern[i] == '"') {+ literal += "\\\"";+ i++;+ } else if (!is_non_literal(sub_pattern[i]) &&+ (i == length - 1 || literal.empty() || sub_pattern[i + 1] == '.' || !is_non_literal(sub_pattern[i + 1]))) {+ literal += sub_pattern[i];+ i++;+ } else {+ break;+ }+ }+ if (!literal.empty()) {+ seq.push_back(std::make_pair(literal, true));+ }+ }+ }+ return join_seq();+ };+ return _add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space");+ }++ std::string _resolve_ref(const std::string & ref) {+ std::string ref_name = ref.substr(ref.find_last_of('/') + 1);+ if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {+ _refs_being_resolved.insert(ref);+ json resolved = _refs[ref];+ ref_name = visit(resolved, ref_name);+ _refs_being_resolved.erase(ref);+ }+ return ref_name;+ }++ std::string _build_object_rule(+ const std::vector<std::pair<std::string, json>> & properties,+ const std::unordered_set<std::string> & required,+ const std::string & name,+ const json & additional_properties)+ {+ std::vector<std::string> required_props;+ std::vector<std::string> optional_props;+ std::unordered_map<std::string, std::string> prop_kv_rule_names;+ for (const auto & kv : properties) {+ const auto &prop_name = kv.first;+ const auto &prop_schema = kv.second;++ std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);+ prop_kv_rule_names[prop_name] = _add_rule(+ name + (name.empty() ? "" : "-") + prop_name + "-kv",+ format_literal(prop_name) + " space \":\" space " + prop_rule_name+ );+ if (required.find(prop_name) != required.end()) {+ required_props.push_back(prop_name);+ } else {+ optional_props.push_back(prop_name);+ }+ }+ if (additional_properties.is_object() || (additional_properties.is_boolean() && additional_properties.get<bool>())) {+ std::string sub_name = name + (name.empty() ? "" : "-") + "additional";+ std::string value_rule = visit(additional_properties.is_object() ? additional_properties : json::object(), sub_name + "-value");+ std::string kv_rule = _add_rule(sub_name + "-kv", _add_rule("string", PRIMITIVE_RULES.at("string")) + " \":\" space " + value_rule);+ prop_kv_rule_names["*"] = kv_rule;+ optional_props.push_back("*");+ }++ std::string rule = "\"{\" space ";+ for (size_t i = 0; i < required_props.size(); i++) {+ if (i > 0) {+ rule += " \",\" space ";+ }+ rule += prop_kv_rule_names[required_props[i]];+ }++ if (!optional_props.empty()) {+ rule += " (";+ if (!required_props.empty()) {+ rule += " \",\" space ( ";+ }++ std::function<std::string(const std::vector<std::string> &, bool)> get_recursive_refs = [&](const std::vector<std::string> & ks, bool first_is_optional) {+ std::string res;+ if (ks.empty()) {+ return res;+ }+ std::string k = ks[0];+ std::string kv_rule_name = prop_kv_rule_names[k];+ if (k == "*") {+ res = _add_rule(+ name + (name.empty() ? "" : "-") + "additional-kvs",+ kv_rule_name + " ( \",\" space " + kv_rule_name + " )*"+ );+ } else if (first_is_optional) {+ res = "( \",\" space " + kv_rule_name + " )?";+ } else {+ res = kv_rule_name;+ }+ if (ks.size() > 1) {+ res += " " + _add_rule(+ name + (name.empty() ? "" : "-") + k + "-rest",+ get_recursive_refs(std::vector<std::string>(ks.begin() + 1, ks.end()), true)+ );+ }+ return res;+ };++ for (size_t i = 0; i < optional_props.size(); i++) {+ if (i > 0) {+ rule += " | ";+ }+ rule += get_recursive_refs(std::vector<std::string>(optional_props.begin() + i, optional_props.end()), false);+ }+ if (!required_props.empty()) {+ rule += " )";+ }+ rule += " )?";+ }++ rule += " \"}\" space";++ return rule;+ }++public:+ SchemaConverter(+ const std::function<json(const std::string &)> & fetch_json,+ bool dotall)+ : _fetch_json(fetch_json), _dotall(dotall)+ {+ _rules["space"] = SPACE_RULE;+ }++ void resolve_refs(nlohmann::json & schema, const std::string & url) {+ /*+ * Resolves all $ref fields in the given schema, fetching any remote schemas,+ * replacing each $ref with absolute reference URL and populates _refs with the+ * respective referenced (sub)schema dictionaries.+ */+ std::function<void(json &)> visit_refs = [&](json & n) {+ if (n.is_array()) {+ for (auto & x : n) {+ visit_refs(x);+ }+ } else if (n.is_object()) {+ if (n.contains("$ref")) {+ std::string ref = n["$ref"];+ if (_refs.find(ref) == _refs.end()) {+ json target;+ if (ref.find("https://") == 0) {+ std::string base_url = ref.substr(0, ref.find('#'));+ auto it = _refs.find(base_url);+ if (it != _refs.end()) {+ target = it->second;+ } else {+ // Fetch the referenced schema and resolve its refs+ auto referenced = _fetch_json(ref);+ resolve_refs(referenced, base_url);+ _refs[base_url] = referenced;+ }+ if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) {+ return;+ }+ } else if (ref.find("#/") == 0) {+ target = schema;+ n["$ref"] = url + ref;+ ref = url + ref;+ } else {+ _errors.push_back("Unsupported ref: " + ref);+ return;+ }+ std::string pointer = ref.substr(ref.find('#') + 1);+ std::vector<std::string> tokens = split(pointer, "/");+ for (size_t i = 1; i < tokens.size(); ++i) {+ std::string sel = tokens[i];+ if (target.is_null() || !target.contains(sel)) {+ _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());+ return;+ }+ target = target[sel];+ }+ _refs[ref] = target;+ }+ } else {+ for (auto & kv : n.items()) {+ visit_refs(kv.value());+ }+ }+ }+ };++ visit_refs(schema);+ }++ std::string _generate_constant_rule(const json & value) {+ if (!value.is_string()) {+ _errors.push_back("Only std::string constants are supported, got " + value.dump());+ return "";+ }+ return format_literal(value.get<std::string>());+ }++ std::string visit(const json & schema, const std::string & name) {+ json schema_type = schema.contains("type") ? schema["type"] : json();+ std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : "";+ std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;++ if (schema.contains("$ref")) {+ return _add_rule(rule_name, _resolve_ref(schema["$ref"]));+ } else if (schema.contains("oneOf") || schema.contains("anyOf")) {+ std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();+ return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));+ } else if (schema_type.is_array()) {+ std::vector<json> schema_types;+ for (const auto & t : schema_type) {+ schema_types.push_back({{"type", t}});+ }+ return _add_rule(rule_name, _generate_union_rule(name, schema_types));+ } else if (schema.contains("const")) {+ return _add_rule(rule_name, _generate_constant_rule(schema["const"]));+ } else if (schema.contains("enum")) {+ std::vector<std::string> enum_values;+ for (const auto & v : schema["enum"]) {+ enum_values.push_back(_generate_constant_rule(v));+ }+ return _add_rule(rule_name, join(enum_values.begin(), enum_values.end(), " | "));+ } else if ((schema_type.is_null() || schema_type == "object")+ && (schema.contains("properties") ||+ (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {+ std::unordered_set<std::string> required;+ if (schema.contains("required") && schema["required"].is_array()) {+ for (const auto & item : schema["required"]) {+ if (item.is_string()) {+ required.insert(item.get<std::string>());+ }+ }+ }+ std::vector<std::pair<std::string, json>> properties;+ if (schema.contains("properties")) {+ for (const auto & prop : schema["properties"].items()) {+ properties.emplace_back(prop.key(), prop.value());+ }+ }+ return _add_rule(rule_name,+ _build_object_rule(+ properties, required, name,+ schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));+ } else if ((schema_type.is_null() || schema_type == "object") && schema.contains("allOf")) {+ std::unordered_set<std::string> required;+ std::vector<std::pair<std::string, json>> properties;+ std::string hybrid_name = name;+ std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) {+ if (comp_schema.contains("$ref")) {+ add_component(_refs[comp_schema["$ref"]], is_required);+ } else if (comp_schema.contains("properties")) {+ for (const auto & prop : comp_schema["properties"].items()) {+ properties.emplace_back(prop.key(), prop.value());+ if (is_required) {+ required.insert(prop.key());+ }+ }+ } else {+ // todo warning+ }+ };+ for (auto & t : schema["allOf"]) {+ if (t.contains("anyOf")) {+ for (auto & tt : t["anyOf"]) {+ add_component(tt, false);+ }+ } else {+ add_component(t, true);+ }+ }+ return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));+ } else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {+ json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];+ if (items.is_array()) {+ std::string rule = "\"[\" space ";+ for (size_t i = 0; i < items.size(); i++) {+ if (i > 0) {+ rule += " \",\" space ";+ }+ rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));+ }+ rule += " \"]\" space";+ return _add_rule(rule_name, rule);+ } else {+ std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");+ std::string list_item_operator = "( \",\" space " + item_rule_name + " )";+ std::string successive_items;+ int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;+ json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();+ int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : -1;+ if (min_items > 0) {+ successive_items += repeat(list_item_operator, min_items - 1);+ min_items--;+ }+ if (max_items >= 0 && max_items > min_items) {+ successive_items += repeat(list_item_operator + "?", max_items - min_items - 1);+ } else {+ successive_items += list_item_operator + "*";+ }+ std::string rule;+ if (min_items == 0) {+ rule = "\"[\" space ( " + item_rule_name + " " + successive_items + " )? \"]\" space";+ } else {+ rule = "\"[\" space " + item_rule_name + " " + successive_items + " \"]\" space";+ }+ return _add_rule(rule_name, rule);+ }+ } else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {+ return _visit_pattern(schema["pattern"], rule_name);+ } else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {+ return _add_rule(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));+ } else if ((schema_type.is_null() || schema_type == "string") && DATE_RULES.find(schema_format) != DATE_RULES.end()) {+ for (const auto & kv : DATE_RULES) {+ _add_rule(kv.first, kv.second);+ }+ return schema_format + "-string";+ } else if (schema.empty() || schema_type == "object") {+ for (const auto & n : OBJECT_RULE_NAMES) {+ _add_rule(n, PRIMITIVE_RULES.at(n));+ }+ return _add_rule(rule_name, "object");+ } else {+ if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {+ _errors.push_back("Unrecognized schema: " + schema.dump());+ return "";+ }+ // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero+ return _add_rule(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>()));+ }+ }++ void check_errors() {+ if (!_errors.empty()) {+ throw std::runtime_error("JSON schema conversion failed:\n" + join(_errors.begin(), _errors.end(), "\n"));+ }+ if (!_warnings.empty()) {+ fprintf(stderr, "WARNING: JSON schema conversion was incomplete: %s\n", join(_warnings.begin(), _warnings.end(), "; ").c_str());+ }+ }++ std::string format_grammar() {+ std::stringstream ss;+ for (const auto & kv : _rules) {+ ss << kv.first << " ::= " << kv.second << std::endl;+ }+ return ss.str();+ }+};++std::string json_schema_to_grammar(const json & schema) {+ SchemaConverter converter([](const std::string &) { return json::object(); }, /* dotall= */ false);+ auto copy = schema;+ converter.resolve_refs(copy, "input");+ converter.visit(copy, "");+ converter.check_errors();+ return converter.format_grammar();+}diff --git a/llama.cpp/json-schema-to-grammar.h b/llama.cpp/json-schema-to-grammar.hnew file mode 100644index 00000000..ad5ff596--- /dev/null+++ b/llama.cpp/json-schema-to-grammar.h@@ -0,0 +1,4 @@+#pragma once+#include "json.h"++std::string json_schema_to_grammar(const nlohmann::json& schema);diff --git a/llama.cpp/server/json.h b/llama.cpp/json.hsimilarity index 93%rename from llama.cpp/server/json.hrename to llama.cpp/json.hindex ea945f34..a858728c 100644--- a/llama.cpp/server/json.h+++ b/llama.cpp/json.h@@ -1,9 +1,9 @@// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT/****************************************************************************\@@ -27,7 +27,6 @@#endif // JSON_NO_IO#include <iterator> // random_access_iterator_tag#include <memory> // unique_ptr-#include <numeric> // accumulate#include <string> // string, stoi, to_string#include <utility> // declval, forward, move, pair, swap#include <vector> // vector@@ -35,10 +34,10 @@// #include <nlohmann/adl_serializer.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -48,10 +47,10 @@// #include <nlohmann/detail/abi_macros.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -60,7 +59,7 @@#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK#if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH)- #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 2+ #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 3#warning "Already included a different version of the library!"#endif#endif@@ -68,7 +67,7 @@#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum)#define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum)-#define NLOHMANN_JSON_VERSION_PATCH 2 // NOLINT(modernize-macro-to-enum)+#define NLOHMANN_JSON_VERSION_PATCH 3 // NOLINT(modernize-macro-to-enum)#ifndef JSON_DIAGNOSTICS#define JSON_DIAGNOSTICS 0@@ -150,10 +149,10 @@// #include <nlohmann/detail/conversions/from_json.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -173,16 +172,19 @@// #include <nlohmann/detail/exceptions.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT#include <cstddef> // nullptr_t#include <exception> // exception+#if JSON_DIAGNOSTICS+ #include <numeric> // accumulate+#endif#include <stdexcept> // runtime_error#include <string> // to_string#include <vector> // vector@@ -190,10 +192,10 @@// #include <nlohmann/detail/value_t.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -206,10 +208,10 @@// #include <nlohmann/detail/macro_scope.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -218,10 +220,10 @@// #include <nlohmann/detail/meta/detected.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -231,10 +233,10 @@// #include <nlohmann/detail/meta/void_t.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -318,10 +320,10 @@ NLOHMANN_JSON_NAMESPACE_END// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-FileCopyrightText: 2016-2021 Evan Nemerson <[email protected]>// SPDX-License-Identifier: MIT@@ -2483,6 +2485,14 @@ JSON_HEDLEY_DIAGNOSTIC_POP#endif#endif+#ifndef JSON_HAS_STATIC_RTTI+ #if !defined(_HAS_STATIC_RTTI) || _HAS_STATIC_RTTI != 0+ #define JSON_HAS_STATIC_RTTI 1+ #else+ #define JSON_HAS_STATIC_RTTI 0+ #endif+#endif+#ifdef JSON_HAS_CPP_17#define JSON_INLINE_VARIABLE inline#else@@ -2590,12 +2600,13 @@ JSON_HEDLEY_DIAGNOSTIC_POPclass NumberUnsignedType, class NumberFloatType, \template<typename> class AllocatorType, \template<typename, typename = void> class JSONSerializer, \- class BinaryType>+ class BinaryType, \+ class CustomBaseClass>#define NLOHMANN_BASIC_JSON_TPL \basic_json<ObjectType, ArrayType, StringType, BooleanType, \NumberIntegerType, NumberUnsignedType, NumberFloatType, \- AllocatorType, JSONSerializer, BinaryType>+ AllocatorType, JSONSerializer, BinaryType, CustomBaseClass>// Macros to simplify conversion from/to types@@ -2745,7 +2756,10 @@ JSON_HEDLEY_DIAGNOSTIC_POP#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \- friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }+ friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }++#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \+ friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) }/*!@brief macro@@ -2756,10 +2770,12 @@ JSON_HEDLEY_DIAGNOSTIC_POPinline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) }+#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \+ inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) }+#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \- inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }-+ inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }// inspired from https://stackoverflow.com/a/26745591// allows to call any std function as if (e.g. with begin):@@ -2923,10 +2939,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/string_escape.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -2998,10 +3014,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/input/position_t.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -3040,10 +3056,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/meta/cpp_future.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-FileCopyrightText: 2018 The Abseil Authors// SPDX-License-Identifier: MIT@@ -3214,10 +3230,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/meta/type_traits.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -3226,14 +3242,15 @@ NLOHMANN_JSON_NAMESPACE_END#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type#include <utility> // declval#include <tuple> // tuple+#include <string> // char_traits// #include <nlohmann/detail/iterators/iterator_traits.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -3298,10 +3315,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/meta/call_std/begin.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -3318,10 +3335,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/meta/call_std/end.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -3342,10 +3359,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/json_fwd.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_@@ -3389,7 +3406,8 @@ NLOHMANN_JSON_NAMESPACE_ENDtemplate<typename U> class AllocatorType = std::allocator,template<typename T, typename SFINAE = void> class JSONSerializer =adl_serializer,- class BinaryType = std::vector<std::uint8_t>>+ class BinaryType = std::vector<std::uint8_t>, // cppcheck-suppress syntaxError+ class CustomBaseClass = void>class basic_json;/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document@@ -3577,6 +3595,63 @@ struct actual_object_comparatortemplate<typename BasicJsonType>using actual_object_comparator_t = typename actual_object_comparator<BasicJsonType>::type;+/////////////////+// char_traits //+/////////////////++// Primary template of char_traits calls std char_traits+template<typename T>+struct char_traits : std::char_traits<T>+{};++// Explicitly define char traits for unsigned char since it is not standard+template<>+struct char_traits<unsigned char> : std::char_traits<char>+{+ using char_type = unsigned char;+ using int_type = uint64_t;++ // Redefine to_int_type function+ static int_type to_int_type(char_type c) noexcept+ {+ return static_cast<int_type>(c);+ }++ static char_type to_char_type(int_type i) noexcept+ {+ return static_cast<char_type>(i);+ }++ static constexpr int_type eof() noexcept+ {+ return static_cast<int_type>(EOF);+ }+};++// Explicitly define char traits for signed char since it is not standard+template<>+struct char_traits<signed char> : std::char_traits<char>+{+ using char_type = signed char;+ using int_type = uint64_t;++ // Redefine to_int_type function+ static int_type to_int_type(char_type c) noexcept+ {+ return static_cast<int_type>(c);+ }++ static char_type to_char_type(int_type i) noexcept+ {+ return static_cast<char_type>(i);+ }++ static constexpr int_type eof() noexcept+ {+ return static_cast<int_type>(EOF);+ }+};+///////////////////// is_ functions /////////////////////@@ -3613,7 +3688,6 @@ template <typename... Ts>struct is_default_constructible<const std::tuple<Ts...>>: conjunction<is_default_constructible<Ts>...> {};-template <typename T, typename... Args>struct is_constructible : std::is_constructible<T, Args...> {};@@ -3629,7 +3703,6 @@ struct is_constructible<std::tuple<Ts...>> : is_default_constructible<std::tupletemplate <typename... Ts>struct is_constructible<const std::tuple<Ts...>> : is_default_constructible<const std::tuple<Ts...>> {};-template<typename T, typename = void>struct is_iterator_traits : std::false_type {};@@ -4039,7 +4112,6 @@ struct value_in_range_of_impl2<OfType, T, false, true>}};-template<typename OfType, typename T>struct value_in_range_of_impl2<OfType, T, true, true>{@@ -4138,10 +4210,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/string_concat.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -4165,28 +4237,28 @@ inline std::size_t concat_length()}template<typename... Args>-inline std::size_t concat_length(const char* cstr, Args&& ... rest);+inline std::size_t concat_length(const char* cstr, const Args& ... rest);template<typename StringType, typename... Args>-inline std::size_t concat_length(const StringType& str, Args&& ... rest);+inline std::size_t concat_length(const StringType& str, const Args& ... rest);template<typename... Args>-inline std::size_t concat_length(const char /*c*/, Args&& ... rest)+inline std::size_t concat_length(const char /*c*/, const Args& ... rest){- return 1 + concat_length(std::forward<Args>(rest)...);+ return 1 + concat_length(rest...);}template<typename... Args>-inline std::size_t concat_length(const char* cstr, Args&& ... rest)+inline std::size_t concat_length(const char* cstr, const Args& ... rest){// cppcheck-suppress ignoredReturnValue- return ::strlen(cstr) + concat_length(std::forward<Args>(rest)...);+ return ::strlen(cstr) + concat_length(rest...);}template<typename StringType, typename... Args>-inline std::size_t concat_length(const StringType& str, Args&& ... rest)+inline std::size_t concat_length(const StringType& str, const Args& ... rest){- return str.size() + concat_length(std::forward<Args>(rest)...);+ return str.size() + concat_length(rest...);}template<typename OutStringType>@@ -4277,7 +4349,7 @@ template<typename OutStringType = std::string, typename... Args>inline OutStringType concat(Args && ... args){OutStringType str;- str.reserve(concat_length(std::forward<Args>(args)...));+ str.reserve(concat_length(args...));concat_into(str, std::forward<Args>(args)...);return str;}@@ -4286,7 +4358,6 @@ inline OutStringType concat(Args && ... args)NLOHMANN_JSON_NAMESPACE_END-NLOHMANN_JSON_NAMESPACE_BEGINnamespace detail{@@ -4334,9 +4405,9 @@ class exception : public std::exception{case value_t::array:{- for (std::size_t i = 0; i < current->m_parent->m_value.array->size(); ++i)+ for (std::size_t i = 0; i < current->m_parent->m_data.m_value.array->size(); ++i){- if (¤t->m_parent->m_value.array->operator[](i) == current)+ if (¤t->m_parent->m_data.m_value.array->operator[](i) == current){tokens.emplace_back(std::to_string(i));break;@@ -4347,7 +4418,7 @@ class exception : public std::exceptioncase value_t::object:{- for (const auto& element : *current->m_parent->m_value.object)+ for (const auto& element : *current->m_parent->m_data.m_value.object){if (&element.second == current){@@ -4410,17 +4481,17 @@ class parse_error : public exceptiontemplate<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>static parse_error create(int id_, const position_t& pos, const std::string& what_arg, BasicJsonContext context){- std::string w = concat(exception::name("parse_error", id_), "parse error",- position_string(pos), ": ", exception::diagnostics(context), what_arg);+ const std::string w = concat(exception::name("parse_error", id_), "parse error",+ position_string(pos), ": ", exception::diagnostics(context), what_arg);return {id_, pos.chars_read_total, w.c_str()};}template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>static parse_error create(int id_, std::size_t byte_, const std::string& what_arg, BasicJsonContext context){- std::string w = concat(exception::name("parse_error", id_), "parse error",- (byte_ != 0 ? (concat(" at byte ", std::to_string(byte_))) : ""),- ": ", exception::diagnostics(context), what_arg);+ const std::string w = concat(exception::name("parse_error", id_), "parse error",+ (byte_ != 0 ? (concat(" at byte ", std::to_string(byte_))) : ""),+ ": ", exception::diagnostics(context), what_arg);return {id_, byte_, w.c_str()};}@@ -4454,7 +4525,7 @@ class invalid_iterator : public exceptiontemplate<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>static invalid_iterator create(int id_, const std::string& what_arg, BasicJsonContext context){- std::string w = concat(exception::name("invalid_iterator", id_), exception::diagnostics(context), what_arg);+ const std::string w = concat(exception::name("invalid_iterator", id_), exception::diagnostics(context), what_arg);return {id_, w.c_str()};}@@ -4472,7 +4543,7 @@ class type_error : public exceptiontemplate<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>static type_error create(int id_, const std::string& what_arg, BasicJsonContext context){- std::string w = concat(exception::name("type_error", id_), exception::diagnostics(context), what_arg);+ const std::string w = concat(exception::name("type_error", id_), exception::diagnostics(context), what_arg);return {id_, w.c_str()};}@@ -4489,7 +4560,7 @@ class out_of_range : public exceptiontemplate<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>static out_of_range create(int id_, const std::string& what_arg, BasicJsonContext context){- std::string w = concat(exception::name("out_of_range", id_), exception::diagnostics(context), what_arg);+ const std::string w = concat(exception::name("out_of_range", id_), exception::diagnostics(context), what_arg);return {id_, w.c_str()};}@@ -4506,7 +4577,7 @@ class other_error : public exceptiontemplate<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>static other_error create(int id_, const std::string& what_arg, BasicJsonContext context){- std::string w = concat(exception::name("other_error", id_), exception::diagnostics(context), what_arg);+ const std::string w = concat(exception::name("other_error", id_), exception::diagnostics(context), what_arg);return {id_, w.c_str()};}@@ -4525,10 +4596,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/meta/identity_tag.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -4549,10 +4620,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/meta/std_fs.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -5055,10 +5126,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/conversions/to_json.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -5075,10 +5146,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/iterators/iteration_proxy.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -5147,10 +5218,10 @@ template<typename IteratorType> class iteration_proxy_value// older GCCs are a bit fussy and require explicit noexcept specifiers on defaulted functionsiteration_proxy_value(iteration_proxy_value&&)noexcept(std::is_nothrow_move_constructible<IteratorType>::value- && std::is_nothrow_move_constructible<string_type>::value) = default;+ && std::is_nothrow_move_constructible<string_type>::value) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor,cppcoreguidelines-noexcept-move-operations)iteration_proxy_value& operator=(iteration_proxy_value&&)noexcept(std::is_nothrow_move_assignable<IteratorType>::value- && std::is_nothrow_move_assignable<string_type>::value) = default;+ && std::is_nothrow_move_assignable<string_type>::value) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor,cppcoreguidelines-noexcept-move-operations)~iteration_proxy_value() = default;/// dereference operator (needed for range-based for)@@ -5297,11 +5368,11 @@ namespace std#pragma clang diagnostic ignored "-Wmismatched-tags"#endiftemplate<typename IteratorType>-class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>>+class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>> // NOLINT(cert-dcl58-cpp): public std::integral_constant<std::size_t, 2> {};template<std::size_t N, typename IteratorType>-class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >>+class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >> // NOLINT(cert-dcl58-cpp){public:using type = decltype(@@ -5340,7 +5411,7 @@ namespace detail/** Note all external_constructor<>::construct functions need to call- * j.m_value.destroy(j.m_type) to avoid a memory leak in case j contains an+ * j.m_data.m_value.destroy(j.m_data.m_type) to avoid a memory leak in case j contains an* allocated value (e.g., a string). See bug issue* https://github.com/nlohmann/json/issues/2865 for more information.*/@@ -5353,9 +5424,9 @@ struct external_constructor<value_t::boolean>template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept{- j.m_value.destroy(j.m_type);- j.m_type = value_t::boolean;- j.m_value = b;+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::boolean;+ j.m_data.m_value = b;j.assert_invariant();}};@@ -5366,18 +5437,18 @@ struct external_constructor<value_t::string>template<typename BasicJsonType>static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s){- j.m_value.destroy(j.m_type);- j.m_type = value_t::string;- j.m_value = s;+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::string;+ j.m_data.m_value = s;j.assert_invariant();}template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s){- j.m_value.destroy(j.m_type);- j.m_type = value_t::string;- j.m_value = std::move(s);+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::string;+ j.m_data.m_value = std::move(s);j.assert_invariant();}@@ -5386,9 +5457,9 @@ struct external_constructor<value_t::string>int > = 0 >static void construct(BasicJsonType& j, const CompatibleStringType& str){- j.m_value.destroy(j.m_type);- j.m_type = value_t::string;- j.m_value.string = j.template create<typename BasicJsonType::string_t>(str);+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::string;+ j.m_data.m_value.string = j.template create<typename BasicJsonType::string_t>(str);j.assert_invariant();}};@@ -5399,18 +5470,18 @@ struct external_constructor<value_t::binary>template<typename BasicJsonType>static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b){- j.m_value.destroy(j.m_type);- j.m_type = value_t::binary;- j.m_value = typename BasicJsonType::binary_t(b);+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::binary;+ j.m_data.m_value = typename BasicJsonType::binary_t(b);j.assert_invariant();}template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b){- j.m_value.destroy(j.m_type);- j.m_type = value_t::binary;- j.m_value = typename BasicJsonType::binary_t(std::move(b));+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::binary;+ j.m_data.m_value = typename BasicJsonType::binary_t(std::move(b));j.assert_invariant();}};@@ -5421,9 +5492,9 @@ struct external_constructor<value_t::number_float>template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept{- j.m_value.destroy(j.m_type);- j.m_type = value_t::number_float;- j.m_value = val;+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::number_float;+ j.m_data.m_value = val;j.assert_invariant();}};@@ -5434,9 +5505,9 @@ struct external_constructor<value_t::number_unsigned>template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept{- j.m_value.destroy(j.m_type);- j.m_type = value_t::number_unsigned;- j.m_value = val;+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::number_unsigned;+ j.m_data.m_value = val;j.assert_invariant();}};@@ -5447,9 +5518,9 @@ struct external_constructor<value_t::number_integer>template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept{- j.m_value.destroy(j.m_type);- j.m_type = value_t::number_integer;- j.m_value = val;+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::number_integer;+ j.m_data.m_value = val;j.assert_invariant();}};@@ -5460,9 +5531,9 @@ struct external_constructor<value_t::array>template<typename BasicJsonType>static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr){- j.m_value.destroy(j.m_type);- j.m_type = value_t::array;- j.m_value = arr;+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::array;+ j.m_data.m_value = arr;j.set_parents();j.assert_invariant();}@@ -5470,9 +5541,9 @@ struct external_constructor<value_t::array>template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr){- j.m_value.destroy(j.m_type);- j.m_type = value_t::array;- j.m_value = std::move(arr);+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::array;+ j.m_data.m_value = std::move(arr);j.set_parents();j.assert_invariant();}@@ -5485,9 +5556,9 @@ struct external_constructor<value_t::array>using std::begin;using std::end;- j.m_value.destroy(j.m_type);- j.m_type = value_t::array;- j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::array;+ j.m_data.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));j.set_parents();j.assert_invariant();}@@ -5495,14 +5566,14 @@ struct external_constructor<value_t::array>template<typename BasicJsonType>static void construct(BasicJsonType& j, const std::vector<bool>& arr){- j.m_value.destroy(j.m_type);- j.m_type = value_t::array;- j.m_value = value_t::array;- j.m_value.array->reserve(arr.size());+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::array;+ j.m_data.m_value = value_t::array;+ j.m_data.m_value.array->reserve(arr.size());for (const bool x : arr){- j.m_value.array->push_back(x);- j.set_parent(j.m_value.array->back());+ j.m_data.m_value.array->push_back(x);+ j.set_parent(j.m_data.m_value.array->back());}j.assert_invariant();}@@ -5511,13 +5582,13 @@ struct external_constructor<value_t::array>enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>static void construct(BasicJsonType& j, const std::valarray<T>& arr){- j.m_value.destroy(j.m_type);- j.m_type = value_t::array;- j.m_value = value_t::array;- j.m_value.array->resize(arr.size());+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::array;+ j.m_data.m_value = value_t::array;+ j.m_data.m_value.array->resize(arr.size());if (arr.size() > 0){- std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin());+ std::copy(std::begin(arr), std::end(arr), j.m_data.m_value.array->begin());}j.set_parents();j.assert_invariant();@@ -5530,9 +5601,9 @@ struct external_constructor<value_t::object>template<typename BasicJsonType>static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj){- j.m_value.destroy(j.m_type);- j.m_type = value_t::object;- j.m_value = obj;+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::object;+ j.m_data.m_value = obj;j.set_parents();j.assert_invariant();}@@ -5540,9 +5611,9 @@ struct external_constructor<value_t::object>template<typename BasicJsonType>static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj){- j.m_value.destroy(j.m_type);- j.m_type = value_t::object;- j.m_value = std::move(obj);+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::object;+ j.m_data.m_value = std::move(obj);j.set_parents();j.assert_invariant();}@@ -5554,9 +5625,9 @@ struct external_constructor<value_t::object>using std::begin;using std::end;- j.m_value.destroy(j.m_type);- j.m_type = value_t::object;- j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));+ j.m_data.m_value.destroy(j.m_data.m_type);+ j.m_data.m_type = value_t::object;+ j.m_data.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));j.set_parents();j.assert_invariant();}@@ -5626,7 +5697,8 @@ template<typename BasicJsonType, typename EnumType,inline void to_json(BasicJsonType& j, EnumType e) noexcept{using underlying_type = typename std::underlying_type<EnumType>::type;- external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));+ static constexpr value_t integral_value_t = std::is_unsigned<underlying_type>::value ? value_t::number_unsigned : value_t::number_integer;+ external_constructor<integral_value_t>::construct(j, static_cast<underlying_type>(e));}#endif // JSON_DISABLE_ENUM_SERIALIZATION@@ -5796,10 +5868,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/byte_container_with_subtype.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -5908,10 +5980,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/hash.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -6041,10 +6113,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/input/binary_reader.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -6067,10 +6139,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/input/input_adapters.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -6094,6 +6166,8 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/macro_scope.hpp>+// #include <nlohmann/detail/meta/type_traits.hpp>+NLOHMANN_JSON_NAMESPACE_BEGINnamespace detail@@ -6140,7 +6214,6 @@ class file_input_adapterstd::FILE* m_file;};-/*!Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark atbeginning of input. Does not support changing the underlying std::streambuf@@ -6214,16 +6287,16 @@ class iterator_input_adapter: current(std::move(first)), end(std::move(last)){}- typename std::char_traits<char_type>::int_type get_character()+ typename char_traits<char_type>::int_type get_character(){if (JSON_HEDLEY_LIKELY(current != end)){- auto result = std::char_traits<char_type>::to_int_type(*current);+ auto result = char_traits<char_type>::to_int_type(*current);std::advance(current, 1);return result;}- return std::char_traits<char_type>::eof();+ return char_traits<char_type>::eof();}private:@@ -6239,7 +6312,6 @@ class iterator_input_adapter}};-template<typename BaseInputAdapter, size_t T>struct wide_string_input_helper;@@ -6363,7 +6435,7 @@ struct wide_string_input_helper<BaseInputAdapter, 2>}};-// Wraps another input apdater to convert wide character types into individual bytes.+// Wraps another input adapter to convert wide character types into individual bytes.template<typename BaseInputAdapter, typename WideCharType>class wide_string_input_adapter{@@ -6408,7 +6480,6 @@ class wide_string_input_adapterstd::size_t utf8_bytes_filled = 0;};-template<typename IteratorType, typename Enable = void>struct iterator_input_adapter_factory{@@ -6565,10 +6636,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/input/json_sax.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -6710,7 +6781,6 @@ struct json_saxvirtual ~json_sax() = default;};-namespace detail{/*!@@ -6812,7 +6882,7 @@ class json_sax_dom_parserJSON_ASSERT(ref_stack.back()->is_object());// add null at given key and store the reference for later- object_element = &(ref_stack.back()->m_value.object->operator[](val));+ object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val));return true;}@@ -6887,8 +6957,8 @@ class json_sax_dom_parserif (ref_stack.back()->is_array()){- ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v));- return &(ref_stack.back()->m_value.array->back());+ ref_stack.back()->m_data.m_value.array->emplace_back(std::forward<Value>(v));+ return &(ref_stack.back()->m_data.m_value.array->back());}JSON_ASSERT(ref_stack.back()->is_object());@@ -7007,7 +7077,7 @@ class json_sax_dom_callback_parser// add discarded value at given key and store the reference for laterif (keep && ref_stack.back()){- object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded);+ object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val) = discarded);}return true;@@ -7092,7 +7162,7 @@ class json_sax_dom_callback_parser// remove discarded valueif (!keep && !ref_stack.empty() && ref_stack.back()->is_array()){- ref_stack.back()->m_value.array->pop_back();+ ref_stack.back()->m_data.m_value.array->pop_back();}return true;@@ -7159,7 +7229,7 @@ class json_sax_dom_callback_parserif (ref_stack.empty()){root = std::move(value);- return {true, &root};+ return {true, & root};}// skip this value if we already decided to skip the parent@@ -7175,8 +7245,8 @@ class json_sax_dom_callback_parser// arrayif (ref_stack.back()->is_array()){- ref_stack.back()->m_value.array->emplace_back(std::move(value));- return {true, &(ref_stack.back()->m_value.array->back())};+ ref_stack.back()->m_data.m_value.array->emplace_back(std::move(value));+ return {true, & (ref_stack.back()->m_data.m_value.array->back())};}// object@@ -7201,9 +7271,9 @@ class json_sax_dom_callback_parser/// stack to model hierarchy of valuesstd::vector<BasicJsonType*> ref_stack {};/// stack to manage which values to keep- std::vector<bool> keep_stack {};+ std::vector<bool> keep_stack {}; // NOLINT(readability-redundant-member-init)/// stack to manage which object keys to keep- std::vector<bool> key_keep_stack {};+ std::vector<bool> key_keep_stack {}; // NOLINT(readability-redundant-member-init)/// helper to hold the reference for the next object elementBasicJsonType* object_element = nullptr;/// whether a syntax error occurred@@ -7298,10 +7368,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/input/lexer.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -7322,6 +7392,8 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/macro_scope.hpp>+// #include <nlohmann/detail/meta/type_traits.hpp>+NLOHMANN_JSON_NAMESPACE_BEGINnamespace detail@@ -7416,7 +7488,7 @@ class lexer : public lexer_base<BasicJsonType>using number_float_t = typename BasicJsonType::number_float_t;using string_t = typename BasicJsonType::string_t;using char_type = typename InputAdapterType::char_type;- using char_int_type = typename std::char_traits<char_type>::int_type;+ using char_int_type = typename char_traits<char_type>::int_type;public:using token_type = typename lexer_base<BasicJsonType>::token_type;@@ -7523,7 +7595,7 @@ class lexer : public lexer_base<BasicJsonType>for (auto range = ranges.begin(); range != ranges.end(); ++range){get();- if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range)))+ if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions){add(current);}@@ -7566,7 +7638,7 @@ class lexer : public lexer_base<BasicJsonType>switch (get()){// end of file while parsing string- case std::char_traits<char_type>::eof():+ case char_traits<char_type>::eof():{error_message = "invalid string: missing closing quote";return token_type::parse_error;@@ -8155,7 +8227,7 @@ class lexer : public lexer_base<BasicJsonType>{case '\n':case '\r':- case std::char_traits<char_type>::eof():+ case char_traits<char_type>::eof():case '\0':return true;@@ -8172,7 +8244,7 @@ class lexer : public lexer_base<BasicJsonType>{switch (get()){- case std::char_traits<char_type>::eof():+ case char_traits<char_type>::eof():case '\0':{error_message = "invalid comment; missing closing '*/'";@@ -8601,10 +8673,10 @@ scan_number_done:token_type scan_literal(const char_type* literal_text, const std::size_t length,token_type return_type){- JSON_ASSERT(std::char_traits<char_type>::to_char_type(current) == literal_text[0]);+ JSON_ASSERT(char_traits<char_type>::to_char_type(current) == literal_text[0]);for (std::size_t i = 1; i < length; ++i){- if (JSON_HEDLEY_UNLIKELY(std::char_traits<char_type>::to_char_type(get()) != literal_text[i]))+ if (JSON_HEDLEY_UNLIKELY(char_traits<char_type>::to_char_type(get()) != literal_text[i])){error_message = "invalid literal";return token_type::parse_error;@@ -8622,7 +8694,7 @@ scan_number_done:{token_buffer.clear();token_string.clear();- token_string.push_back(std::char_traits<char_type>::to_char_type(current));+ token_string.push_back(char_traits<char_type>::to_char_type(current));}/*@@ -8630,7 +8702,7 @@ scan_number_done:This function provides the interface to the used input adapter. It doesnot throw in case the input reached EOF, but returns a- `std::char_traits<char>::eof()` in that case. Stores the scanned characters+ `char_traits<char>::eof()` in that case. Stores the scanned charactersfor use in error messages.@return character read from the input@@ -8650,9 +8722,9 @@ scan_number_done:current = ia.get_character();}- if (JSON_HEDLEY_LIKELY(current != std::char_traits<char_type>::eof()))+ if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof())){- token_string.push_back(std::char_traits<char_type>::to_char_type(current));+ token_string.push_back(char_traits<char_type>::to_char_type(current));}if (current == '\n')@@ -8691,7 +8763,7 @@ scan_number_done:--position.chars_read_current_line;}- if (JSON_HEDLEY_LIKELY(current != std::char_traits<char_type>::eof()))+ if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof())){JSON_ASSERT(!token_string.empty());token_string.pop_back();@@ -8885,7 +8957,7 @@ scan_number_done:// end of input (the null byte is needed when parsing from// string literals)case '\0':- case std::char_traits<char_type>::eof():+ case char_traits<char_type>::eof():return token_type::end_of_input;// error@@ -8903,7 +8975,7 @@ scan_number_done:const bool ignore_comments = false;/// the current character- char_int_type current = std::char_traits<char_type>::eof();+ char_int_type current = char_traits<char_type>::eof();/// whether the next get() call should just return currentbool next_unget = false;@@ -8937,10 +9009,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/meta/is_sax.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -9129,7 +9201,6 @@ static inline bool little_endianness(int num = 1) noexceptreturn *reinterpret_cast<char*>(&num) == 1;}-///////////////////// binary reader /////////////////////@@ -9147,7 +9218,7 @@ class binary_readerusing binary_t = typename BasicJsonType::binary_t;using json_sax_t = SAX;using char_type = typename InputAdapterType::char_type;- using char_int_type = typename std::char_traits<char_type>::int_type;+ using char_int_type = typename char_traits<char_type>::int_type;public:/*!@@ -9220,7 +9291,7 @@ class binary_readerget();}- if (JSON_HEDLEY_UNLIKELY(current != std::char_traits<char_type>::eof()))+ if (JSON_HEDLEY_UNLIKELY(current != char_traits<char_type>::eof())){return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read,exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr));@@ -9303,7 +9374,7 @@ class binary_readerexception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr));}- return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) && get() != std::char_traits<char_type>::eof();+ return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) && get() != char_traits<char_type>::eof();}/*!@@ -9404,7 +9475,7 @@ class binary_reader{std::array<char, 3> cr{{}};static_cast<void>((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast<unsigned char>(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)- std::string cr_str{cr.data()};+ const std::string cr_str{cr.data()};return sax->parse_error(element_type_parse_position, cr_str,parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr));}@@ -9497,7 +9568,7 @@ class binary_readerswitch (get_char ? get() : current){// EOF- case std::char_traits<char_type>::eof():+ case char_traits<char_type>::eof():return unexpect_eof(input_format_t::cbor, "value");// Integer 0x00..0x17 (0..23)@@ -10272,7 +10343,7 @@ class binary_readerswitch (get()){// EOF- case std::char_traits<char_type>::eof():+ case char_traits<char_type>::eof():return unexpect_eof(input_format_t::msgpack, "value");// positive fixint@@ -11339,7 +11410,7 @@ class binary_readerexception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr));}- bool is_error = get_ubjson_size_value(result.first, is_ndarray);+ const bool is_error = get_ubjson_size_value(result.first, is_ndarray);if (input_format == input_format_t::bjdata && is_ndarray){if (inside_ndarray)@@ -11354,7 +11425,7 @@ class binary_readerif (current == '#'){- bool is_error = get_ubjson_size_value(result.first, is_ndarray);+ const bool is_error = get_ubjson_size_value(result.first, is_ndarray);if (input_format == input_format_t::bjdata && is_ndarray){return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,@@ -11374,7 +11445,7 @@ class binary_reader{switch (prefix){- case std::char_traits<char_type>::eof(): // EOF+ case char_traits<char_type>::eof(): // EOFreturn unexpect_eof(input_format, "value");case 'T': // true@@ -11819,7 +11890,7 @@ class binary_readerThis function provides the interface to the used input adapter. It doesnot throw in case the input reached EOF, but returns a -'ve valued- `std::char_traits<char_type>::eof()` in that case.+ `char_traits<char_type>::eof()` in that case.@return character read from the input*/@@ -11961,7 +12032,7 @@ class binary_readerJSON_HEDLEY_NON_NULL(3)bool unexpect_eof(const input_format_t format, const char* context) const{- if (JSON_HEDLEY_UNLIKELY(current == std::char_traits<char_type>::eof()))+ if (JSON_HEDLEY_UNLIKELY(current == char_traits<char_type>::eof())){return sax->parse_error(chars_read, "<end of file>",parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr));@@ -12028,7 +12099,7 @@ class binary_readerInputAdapterType ia;/// the current character- char_int_type current = std::char_traits<char_type>::eof();+ char_int_type current = char_traits<char_type>::eof();/// the number of characters readstd::size_t chars_read = 0;@@ -12090,10 +12161,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/input/parser.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -12439,13 +12510,25 @@ class parserm_lexer.get_token_string(),parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr));}+ case token_type::end_of_input:+ {+ if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))+ {+ return sax->parse_error(m_lexer.get_position(),+ m_lexer.get_token_string(),+ parse_error::create(101, m_lexer.get_position(),+ "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr));+ }+ return sax->parse_error(m_lexer.get_position(),+ m_lexer.get_token_string(),+ parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));+ }case token_type::uninitialized:case token_type::end_array:case token_type::end_object:case token_type::name_separator:case token_type::value_separator:- case token_type::end_of_input:case token_type::literal_or_value:default: // the last token was unexpected{@@ -12607,10 +12690,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/iterators/internal_iterator.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -12620,10 +12703,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/iterators/primitive_iterator.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -12779,10 +12862,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/iterators/iter_impl.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -12887,7 +12970,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:{@@ -12984,17 +13067,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:{- m_it.object_iterator = m_object->m_value.object->begin();+ m_it.object_iterator = m_object->m_data.m_value.object->begin();break;}case value_t::array:{- m_it.array_iterator = m_object->m_value.array->begin();+ m_it.array_iterator = m_object->m_data.m_value.array->begin();break;}@@ -13028,17 +13111,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:{- m_it.object_iterator = m_object->m_value.object->end();+ m_it.object_iterator = m_object->m_data.m_value.object->end();break;}case value_t::array:{- m_it.array_iterator = m_object->m_value.array->end();+ m_it.array_iterator = m_object->m_data.m_value.array->end();break;}@@ -13067,17 +13150,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:{- JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end());+ JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end());return m_it.object_iterator->second;}case value_t::array:{- JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end());+ JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end());return *m_it.array_iterator;}@@ -13111,17 +13194,17 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:{- JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end());+ JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end());return &(m_it.object_iterator->second);}case value_t::array:{- JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end());+ JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end());return &*m_it.array_iterator;}@@ -13164,7 +13247,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:{@@ -13215,7 +13298,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:{@@ -13262,7 +13345,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speciJSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:return (m_it.object_iterator == other.m_it.object_iterator);@@ -13307,7 +13390,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speciJSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators", m_object));@@ -13363,7 +13446,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", m_object));@@ -13442,7 +13525,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", m_object));@@ -13471,7 +13554,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci{JSON_ASSERT(m_object != nullptr);- switch (m_object->m_type)+ switch (m_object->m_data.m_type){case value_t::object:JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators", m_object));@@ -13541,10 +13624,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -13673,13 +13756,55 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/iterators/primitive_iterator.hpp>+// #include <nlohmann/detail/json_custom_base_class.hpp>+// __ _____ _____ _____+// __| | __| | | | JSON for Modern C+++// | | |__ | | | | | | version 3.11.3+// |_____|_____|_____|_|___| https://github.com/nlohmann/json+//+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>+// SPDX-License-Identifier: MIT++++#include <type_traits> // conditional, is_same++// #include <nlohmann/detail/abi_macros.hpp>+++NLOHMANN_JSON_NAMESPACE_BEGIN+namespace detail+{++/*!+@brief Default base class of the @ref basic_json class.++So that the correct implementations of the copy / move ctors / assign operators+of @ref basic_json do not require complex case distinctions+(no base class / custom base class used as customization point),+@ref basic_json always has a base class.+By default, this class is used because it is empty and thus has no effect+on the behavior of @ref basic_json.+*/+struct json_default_base {};++template<class T>+using json_base_class = typename std::conditional <+ std::is_same<T, void>::value,+ json_default_base,+ T+ >::type;++} // namespace detail+NLOHMANN_JSON_NAMESPACE_END+// #include <nlohmann/detail/json_pointer.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -13911,7 +14036,7 @@ class json_pointerconst char* p = s.c_str();char* p_end = nullptr;errno = 0; // strtoull doesn't reset errno- unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int)+ const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int)if (p == p_end // invalid input or empty string|| errno == ERANGE // out of range|| JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(p_end - p) != s.size())) // incomplete read@@ -14067,7 +14192,7 @@ class json_pointerif (reference_token == "-"){// explicitly treat "-" as index beyond the end- ptr = &ptr->operator[](ptr->m_value.array->size());+ ptr = &ptr->operator[](ptr->m_data.m_value.array->size());}else{@@ -14119,7 +14244,7 @@ class json_pointer{// "-" always fails the range checkJSON_THROW(detail::out_of_range::create(402, detail::concat(- "array index '-' (", std::to_string(ptr->m_value.array->size()),+ "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()),") is out of range"), ptr));}@@ -14176,7 +14301,7 @@ class json_pointerif (JSON_HEDLEY_UNLIKELY(reference_token == "-")){// "-" cannot be used for const access- JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_value.array->size()), ") is out of range"), ptr));+ JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr));}// use unchecked array access@@ -14226,7 +14351,7 @@ class json_pointer{// "-" always fails the range checkJSON_THROW(detail::out_of_range::create(402, detail::concat(- "array index '-' (", std::to_string(ptr->m_value.array->size()),+ "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()),") is out of range"), ptr));}@@ -14421,7 +14546,7 @@ class json_pointer{case detail::value_t::array:{- if (value.m_value.array->empty())+ if (value.m_data.m_value.array->empty()){// flatten empty array as nullresult[reference_string] = nullptr;@@ -14429,10 +14554,10 @@ class json_pointerelse{// iterate array and use index as reference string- for (std::size_t i = 0; i < value.m_value.array->size(); ++i)+ for (std::size_t i = 0; i < value.m_data.m_value.array->size(); ++i){flatten(detail::concat(reference_string, '/', std::to_string(i)),- value.m_value.array->operator[](i), result);+ value.m_data.m_value.array->operator[](i), result);}}break;@@ -14440,7 +14565,7 @@ class json_pointercase detail::value_t::object:{- if (value.m_value.object->empty())+ if (value.m_data.m_value.object->empty()){// flatten empty object as nullresult[reference_string] = nullptr;@@ -14448,7 +14573,7 @@ class json_pointerelse{// iterate object and use keys as reference string- for (const auto& element : *value.m_value.object)+ for (const auto& element : *value.m_data.m_value.object){flatten(detail::concat(reference_string, '/', detail::escape(element.first)), element.second, result);}@@ -14495,7 +14620,7 @@ class json_pointerBasicJsonType result;// iterate the JSON object values- for (const auto& element : *value.m_value.object)+ for (const auto& element : *value.m_data.m_value.object){if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())){@@ -14671,10 +14796,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/json_ref.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -14763,10 +14888,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/output/binary_writer.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -14789,10 +14914,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/output/output_adapters.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -14978,7 +15103,7 @@ class binary_writer{case value_t::object:{- write_bson_object(*j.m_value.object);+ write_bson_object(*j.m_data.m_value.object);break;}@@ -15013,7 +15138,7 @@ class binary_writercase value_t::boolean:{- oa->write_character(j.m_value.boolean+ oa->write_character(j.m_data.m_value.boolean? to_char_type(0xF5): to_char_type(0xF4));break;@@ -15021,42 +15146,42 @@ class binary_writercase value_t::number_integer:{- if (j.m_value.number_integer >= 0)+ if (j.m_data.m_value.number_integer >= 0){// CBOR does not differentiate between positive signed// integers and unsigned integers. Therefore, we used the// code from the value_t::number_unsigned case here.- if (j.m_value.number_integer <= 0x17)+ if (j.m_data.m_value.number_integer <= 0x17){- write_number(static_cast<std::uint8_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)())+ else if (j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)()){oa->write_character(to_char_type(0x18));- write_number(static_cast<std::uint8_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)())+ else if (j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)()){oa->write_character(to_char_type(0x19));- write_number(static_cast<std::uint16_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)())+ else if (j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)()){oa->write_character(to_char_type(0x1A));- write_number(static_cast<std::uint32_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));}else{oa->write_character(to_char_type(0x1B));- write_number(static_cast<std::uint64_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));}}else{// The conversions below encode the sign in the first// byte, and the value is converted to a positive number.- const auto positive_number = -1 - j.m_value.number_integer;- if (j.m_value.number_integer >= -24)+ const auto positive_number = -1 - j.m_data.m_value.number_integer;+ if (j.m_data.m_value.number_integer >= -24){write_number(static_cast<std::uint8_t>(0x20 + positive_number));}@@ -15086,52 +15211,52 @@ class binary_writercase value_t::number_unsigned:{- if (j.m_value.number_unsigned <= 0x17)+ if (j.m_data.m_value.number_unsigned <= 0x17){- write_number(static_cast<std::uint8_t>(j.m_value.number_unsigned));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)()){oa->write_character(to_char_type(0x18));- write_number(static_cast<std::uint8_t>(j.m_value.number_unsigned));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)()){oa->write_character(to_char_type(0x19));- write_number(static_cast<std::uint16_t>(j.m_value.number_unsigned));+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_unsigned));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)()){oa->write_character(to_char_type(0x1A));- write_number(static_cast<std::uint32_t>(j.m_value.number_unsigned));+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_unsigned));}else{oa->write_character(to_char_type(0x1B));- write_number(static_cast<std::uint64_t>(j.m_value.number_unsigned));+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));}break;}case value_t::number_float:{- if (std::isnan(j.m_value.number_float))+ if (std::isnan(j.m_data.m_value.number_float)){// NaN is 0xf97e00 in CBORoa->write_character(to_char_type(0xF9));oa->write_character(to_char_type(0x7E));oa->write_character(to_char_type(0x00));}- else if (std::isinf(j.m_value.number_float))+ else if (std::isinf(j.m_data.m_value.number_float)){// Infinity is 0xf97c00, -Infinity is 0xf9fc00oa->write_character(to_char_type(0xf9));- oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC));+ oa->write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC));oa->write_character(to_char_type(0x00));}else{- write_compact_float(j.m_value.number_float, detail::input_format_t::cbor);+ write_compact_float(j.m_data.m_value.number_float, detail::input_format_t::cbor);}break;}@@ -15139,7 +15264,7 @@ class binary_writercase value_t::string:{// step 1: write control byte and the string length- const auto N = j.m_value.string->size();+ const auto N = j.m_data.m_value.string->size();if (N <= 0x17){write_number(static_cast<std::uint8_t>(0x60 + N));@@ -15169,15 +15294,15 @@ class binary_writer// step 2: write the stringoa->write_characters(- reinterpret_cast<const CharType*>(j.m_value.string->c_str()),- j.m_value.string->size());+ reinterpret_cast<const CharType*>(j.m_data.m_value.string->c_str()),+ j.m_data.m_value.string->size());break;}case value_t::array:{// step 1: write control byte and the array size- const auto N = j.m_value.array->size();+ const auto N = j.m_data.m_value.array->size();if (N <= 0x17){write_number(static_cast<std::uint8_t>(0x80 + N));@@ -15206,7 +15331,7 @@ class binary_writer// LCOV_EXCL_STOP// step 2: write each element- for (const auto& el : *j.m_value.array)+ for (const auto& el : *j.m_data.m_value.array){write_cbor(el);}@@ -15215,32 +15340,32 @@ class binary_writercase value_t::binary:{- if (j.m_value.binary->has_subtype())+ if (j.m_data.m_value.binary->has_subtype()){- if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint8_t>::max)())+ if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint8_t>::max)()){write_number(static_cast<std::uint8_t>(0xd8));- write_number(static_cast<std::uint8_t>(j.m_value.binary->subtype()));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.binary->subtype()));}- else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint16_t>::max)())+ else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint16_t>::max)()){write_number(static_cast<std::uint8_t>(0xd9));- write_number(static_cast<std::uint16_t>(j.m_value.binary->subtype()));+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.binary->subtype()));}- else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint32_t>::max)())+ else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint32_t>::max)()){write_number(static_cast<std::uint8_t>(0xda));- write_number(static_cast<std::uint32_t>(j.m_value.binary->subtype()));+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.binary->subtype()));}- else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint64_t>::max)())+ else if (j.m_data.m_value.binary->subtype() <= (std::numeric_limits<std::uint64_t>::max)()){write_number(static_cast<std::uint8_t>(0xdb));- write_number(static_cast<std::uint64_t>(j.m_value.binary->subtype()));+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.binary->subtype()));}}// step 1: write control byte and the binary array size- const auto N = j.m_value.binary->size();+ const auto N = j.m_data.m_value.binary->size();if (N <= 0x17){write_number(static_cast<std::uint8_t>(0x40 + N));@@ -15270,7 +15395,7 @@ class binary_writer// step 2: write each elementoa->write_characters(- reinterpret_cast<const CharType*>(j.m_value.binary->data()),+ reinterpret_cast<const CharType*>(j.m_data.m_value.binary->data()),N);break;@@ -15279,7 +15404,7 @@ class binary_writercase value_t::object:{// step 1: write control byte and the object size- const auto N = j.m_value.object->size();+ const auto N = j.m_data.m_value.object->size();if (N <= 0x17){write_number(static_cast<std::uint8_t>(0xA0 + N));@@ -15308,7 +15433,7 @@ class binary_writer// LCOV_EXCL_STOP// step 2: write each element- for (const auto& el : *j.m_value.object)+ for (const auto& el : *j.m_data.m_value.object){write_cbor(el.first);write_cbor(el.second);@@ -15337,7 +15462,7 @@ class binary_writercase value_t::boolean: // true and false{- oa->write_character(j.m_value.boolean+ oa->write_character(j.m_data.m_value.boolean? to_char_type(0xC3): to_char_type(0xC2));break;@@ -15345,75 +15470,75 @@ class binary_writercase value_t::number_integer:{- if (j.m_value.number_integer >= 0)+ if (j.m_data.m_value.number_integer >= 0){// MessagePack does not differentiate between positive// signed integers and unsigned integers. Therefore, we used// the code from the value_t::number_unsigned case here.- if (j.m_value.number_unsigned < 128)+ if (j.m_data.m_value.number_unsigned < 128){// positive fixnum- write_number(static_cast<std::uint8_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)()){// uint 8oa->write_character(to_char_type(0xCC));- write_number(static_cast<std::uint8_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)()){// uint 16oa->write_character(to_char_type(0xCD));- write_number(static_cast<std::uint16_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)()){// uint 32oa->write_character(to_char_type(0xCE));- write_number(static_cast<std::uint32_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)()){// uint 64oa->write_character(to_char_type(0xCF));- write_number(static_cast<std::uint64_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));}}else{- if (j.m_value.number_integer >= -32)+ if (j.m_data.m_value.number_integer >= -32){// negative fixnum- write_number(static_cast<std::int8_t>(j.m_value.number_integer));+ write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&- j.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)()){// int 8oa->write_character(to_char_type(0xD0));- write_number(static_cast<std::int8_t>(j.m_value.number_integer));+ write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&- j.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)()){// int 16oa->write_character(to_char_type(0xD1));- write_number(static_cast<std::int16_t>(j.m_value.number_integer));+ write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&- j.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)()){// int 32oa->write_character(to_char_type(0xD2));- write_number(static_cast<std::int32_t>(j.m_value.number_integer));+ write_number(static_cast<std::int32_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_integer >= (std::numeric_limits<std::int64_t>::min)() &&- j.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)())+ else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int64_t>::min)() &&+ j.m_data.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)()){// int 64oa->write_character(to_char_type(0xD3));- write_number(static_cast<std::int64_t>(j.m_value.number_integer));+ write_number(static_cast<std::int64_t>(j.m_data.m_value.number_integer));}}break;@@ -15421,48 +15546,48 @@ class binary_writercase value_t::number_unsigned:{- if (j.m_value.number_unsigned < 128)+ if (j.m_data.m_value.number_unsigned < 128){// positive fixnum- write_number(static_cast<std::uint8_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)()){// uint 8oa->write_character(to_char_type(0xCC));- write_number(static_cast<std::uint8_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)()){// uint 16oa->write_character(to_char_type(0xCD));- write_number(static_cast<std::uint16_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)()){// uint 32oa->write_character(to_char_type(0xCE));- write_number(static_cast<std::uint32_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));}- else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())+ else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)()){// uint 64oa->write_character(to_char_type(0xCF));- write_number(static_cast<std::uint64_t>(j.m_value.number_integer));+ write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));}break;}case value_t::number_float:{- write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack);+ write_compact_float(j.m_data.m_value.number_float, detail::input_format_t::msgpack);break;}case value_t::string:{// step 1: write control byte and the string length- const auto N = j.m_value.string->size();+ const auto N = j.m_data.m_value.string->size();if (N <= 31){// fixstr@@ -15489,15 +15614,15 @@ class binary_writer// step 2: write the stringoa->write_characters(- reinterpret_cast<const CharType*>(j.m_value.string->c_str()),- j.m_value.string->size());+ reinterpret_cast<const CharType*>(j.m_data.m_value.string->c_str()),+ j.m_data.m_value.string->size());break;}case value_t::array:{// step 1: write control byte and the array size- const auto N = j.m_value.array->size();+ const auto N = j.m_data.m_value.array->size();if (N <= 15){// fixarray@@ -15517,7 +15642,7 @@ class binary_writer}// step 2: write each element- for (const auto& el : *j.m_value.array)+ for (const auto& el : *j.m_data.m_value.array){write_msgpack(el);}@@ -15528,10 +15653,10 @@ class binary_writer{// step 0: determine if the binary type has a set subtype to// determine whether or not to use the ext or fixext types- const bool use_ext = j.m_value.binary->has_subtype();+ const bool use_ext = j.m_data.m_value.binary->has_subtype();// step 1: write control byte and the byte string length- const auto N = j.m_value.binary->size();+ const auto N = j.m_data.m_value.binary->size();if (N <= (std::numeric_limits<std::uint8_t>::max)()){std::uint8_t output_type{};@@ -15576,18 +15701,18 @@ class binary_writer}else if (N <= (std::numeric_limits<std::uint16_t>::max)()){- std::uint8_t output_type = use_ext- ? 0xC8 // ext 16- : 0xC5; // bin 16+ const std::uint8_t output_type = use_ext+ ? 0xC8 // ext 16+ : 0xC5; // bin 16oa->write_character(to_char_type(output_type));write_number(static_cast<std::uint16_t>(N));}else if (N <= (std::numeric_limits<std::uint32_t>::max)()){- std::uint8_t output_type = use_ext- ? 0xC9 // ext 32- : 0xC6; // bin 32+ const std::uint8_t output_type = use_ext+ ? 0xC9 // ext 32+ : 0xC6; // bin 32oa->write_character(to_char_type(output_type));write_number(static_cast<std::uint32_t>(N));@@ -15596,12 +15721,12 @@ class binary_writer// step 1.5: if this is an ext type, write the subtypeif (use_ext){- write_number(static_cast<std::int8_t>(j.m_value.binary->subtype()));+ write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));}// step 2: write the byte stringoa->write_characters(- reinterpret_cast<const CharType*>(j.m_value.binary->data()),+ reinterpret_cast<const CharType*>(j.m_data.m_value.binary->data()),N);break;@@ -15610,7 +15735,7 @@ class binary_writercase value_t::object:{// step 1: write control byte and the object size- const auto N = j.m_value.object->size();+ const auto N = j.m_data.m_value.object->size();if (N <= 15){// fixmap@@ -15630,7 +15755,7 @@ class binary_writer}// step 2: write each element- for (const auto& el : *j.m_value.object)+ for (const auto& el : *j.m_data.m_value.object){write_msgpack(el.first);write_msgpack(el.second);@@ -15670,7 +15795,7 @@ class binary_writer{if (add_prefix){- oa->write_character(j.m_value.boolean+ oa->write_character(j.m_data.m_value.boolean? to_char_type('T'): to_char_type('F'));}@@ -15679,19 +15804,19 @@ class binary_writercase value_t::number_integer:{- write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix, use_bjdata);+ write_number_with_ubjson_prefix(j.m_data.m_value.number_integer, add_prefix, use_bjdata);break;}case value_t::number_unsigned:{- write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix, use_bjdata);+ write_number_with_ubjson_prefix(j.m_data.m_value.number_unsigned, add_prefix, use_bjdata);break;}case value_t::number_float:{- write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix, use_bjdata);+ write_number_with_ubjson_prefix(j.m_data.m_value.number_float, add_prefix, use_bjdata);break;}@@ -15701,10 +15826,10 @@ class binary_writer{oa->write_character(to_char_type('S'));}- write_number_with_ubjson_prefix(j.m_value.string->size(), true, use_bjdata);+ write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata);oa->write_characters(- reinterpret_cast<const CharType*>(j.m_value.string->c_str()),- j.m_value.string->size());+ reinterpret_cast<const CharType*>(j.m_data.m_value.string->c_str()),+ j.m_data.m_value.string->size());break;}@@ -15716,7 +15841,7 @@ class binary_writer}bool prefix_required = true;- if (use_type && !j.m_value.array->empty())+ if (use_type && !j.m_data.m_value.array->empty()){JSON_ASSERT(use_count);const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata);@@ -15739,10 +15864,10 @@ class binary_writerif (use_count){oa->write_character(to_char_type('#'));- write_number_with_ubjson_prefix(j.m_value.array->size(), true, use_bjdata);+ write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata);}- for (const auto& el : *j.m_value.array)+ for (const auto& el : *j.m_data.m_value.array){write_ubjson(el, use_count, use_type, prefix_required, use_bjdata);}@@ -15762,7 +15887,7 @@ class binary_writeroa->write_character(to_char_type('['));}- if (use_type && !j.m_value.binary->empty())+ if (use_type && !j.m_data.m_value.binary->empty()){JSON_ASSERT(use_count);oa->write_character(to_char_type('$'));@@ -15772,21 +15897,21 @@ class binary_writerif (use_count){oa->write_character(to_char_type('#'));- write_number_with_ubjson_prefix(j.m_value.binary->size(), true, use_bjdata);+ write_number_with_ubjson_prefix(j.m_data.m_value.binary->size(), true, use_bjdata);}if (use_type){oa->write_characters(- reinterpret_cast<const CharType*>(j.m_value.binary->data()),- j.m_value.binary->size());+ reinterpret_cast<const CharType*>(j.m_data.m_value.binary->data()),+ j.m_data.m_value.binary->size());}else{- for (size_t i = 0; i < j.m_value.binary->size(); ++i)+ for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i){oa->write_character(to_char_type('U'));- oa->write_character(j.m_value.binary->data()[i]);+ oa->write_character(j.m_data.m_value.binary->data()[i]);}}@@ -15800,9 +15925,9 @@ class binary_writercase value_t::object:{- if (use_bjdata && j.m_value.object->size() == 3 && j.m_value.object->find("_ArrayType_") != j.m_value.object->end() && j.m_value.object->find("_ArraySize_") != j.m_value.object->end() && j.m_value.object->find("_ArrayData_") != j.m_value.object->end())+ if (use_bjdata && j.m_data.m_value.object->size() == 3 && j.m_data.m_value.object->find("_ArrayType_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArraySize_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArrayData_") != j.m_data.m_value.object->end()){- if (!write_bjdata_ndarray(*j.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata)+ if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata){break;}@@ -15814,7 +15939,7 @@ class binary_writer}bool prefix_required = true;- if (use_type && !j.m_value.object->empty())+ if (use_type && !j.m_data.m_value.object->empty()){JSON_ASSERT(use_count);const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata);@@ -15837,10 +15962,10 @@ class binary_writerif (use_count){oa->write_character(to_char_type('#'));- write_number_with_ubjson_prefix(j.m_value.object->size(), true, use_bjdata);+ write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata);}- for (const auto& el : *j.m_value.object)+ for (const auto& el : *j.m_data.m_value.object){write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata);oa->write_characters(@@ -15990,19 +16115,19 @@ class binary_writervoid write_bson_unsigned(const string_t& name,const BasicJsonType& j){- if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())){write_bson_entry_header(name, 0x10 /* int32 */);- write_number<std::int32_t>(static_cast<std::int32_t>(j.m_value.number_unsigned), true);+ write_number<std::int32_t>(static_cast<std::int32_t>(j.m_data.m_value.number_unsigned), true);}- else if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))+ else if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)())){write_bson_entry_header(name, 0x12 /* int64 */);- write_number<std::int64_t>(static_cast<std::int64_t>(j.m_value.number_unsigned), true);+ write_number<std::int64_t>(static_cast<std::int64_t>(j.m_data.m_value.number_unsigned), true);}else{- JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_value.number_unsigned), " cannot be represented by BSON as it does not fit int64"), &j));+ JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BSON as it does not fit int64"), &j));}}@@ -16083,13 +16208,13 @@ class binary_writerswitch (j.type()){case value_t::object:- return header_size + calc_bson_object_size(*j.m_value.object);+ return header_size + calc_bson_object_size(*j.m_data.m_value.object);case value_t::array:- return header_size + calc_bson_array_size(*j.m_value.array);+ return header_size + calc_bson_array_size(*j.m_data.m_value.array);case value_t::binary:- return header_size + calc_bson_binary_size(*j.m_value.binary);+ return header_size + calc_bson_binary_size(*j.m_data.m_value.binary);case value_t::boolean:return header_size + 1ul;@@ -16098,13 +16223,13 @@ class binary_writerreturn header_size + 8ul;case value_t::number_integer:- return header_size + calc_bson_integer_size(j.m_value.number_integer);+ return header_size + calc_bson_integer_size(j.m_data.m_value.number_integer);case value_t::number_unsigned:- return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned);+ return header_size + calc_bson_unsigned_size(j.m_data.m_value.number_unsigned);case value_t::string:- return header_size + calc_bson_string_size(*j.m_value.string);+ return header_size + calc_bson_string_size(*j.m_data.m_value.string);case value_t::null:return header_size + 0ul;@@ -16130,28 +16255,28 @@ class binary_writerswitch (j.type()){case value_t::object:- return write_bson_object_entry(name, *j.m_value.object);+ return write_bson_object_entry(name, *j.m_data.m_value.object);case value_t::array:- return write_bson_array(name, *j.m_value.array);+ return write_bson_array(name, *j.m_data.m_value.array);case value_t::binary:- return write_bson_binary(name, *j.m_value.binary);+ return write_bson_binary(name, *j.m_data.m_value.binary);case value_t::boolean:- return write_bson_boolean(name, j.m_value.boolean);+ return write_bson_boolean(name, j.m_data.m_value.boolean);case value_t::number_float:- return write_bson_double(name, j.m_value.number_float);+ return write_bson_double(name, j.m_data.m_value.number_float);case value_t::number_integer:- return write_bson_integer(name, j.m_value.number_integer);+ return write_bson_integer(name, j.m_data.m_value.number_integer);case value_t::number_unsigned:return write_bson_unsigned(name, j);case value_t::string:- return write_bson_string(name, *j.m_value.string);+ return write_bson_string(name, *j.m_data.m_value.string);case value_t::null:return write_bson_null(name);@@ -16173,8 +16298,8 @@ class binary_writer*/static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value){- std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast<std::size_t>(0),- [](size_t result, const typename BasicJsonType::object_t::value_type & el)+ const std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast<std::size_t>(0),+ [](size_t result, const typename BasicJsonType::object_t::value_type & el){return result += calc_bson_element_size(el.first, el.second);});@@ -16424,35 +16549,35 @@ class binary_writerreturn 'Z';case value_t::boolean:- return j.m_value.boolean ? 'T' : 'F';+ return j.m_data.m_value.boolean ? 'T' : 'F';case value_t::number_integer:{- if ((std::numeric_limits<std::int8_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())+ if ((std::numeric_limits<std::int8_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)()){return 'i';}- if ((std::numeric_limits<std::uint8_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)())+ if ((std::numeric_limits<std::uint8_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)()){return 'U';}- if ((std::numeric_limits<std::int16_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())+ if ((std::numeric_limits<std::int16_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)()){return 'I';}- if (use_bjdata && ((std::numeric_limits<std::uint16_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)()))+ if (use_bjdata && ((std::numeric_limits<std::uint16_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)())){return 'u';}- if ((std::numeric_limits<std::int32_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())+ if ((std::numeric_limits<std::int32_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)()){return 'l';}- if (use_bjdata && ((std::numeric_limits<std::uint32_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)()))+ if (use_bjdata && ((std::numeric_limits<std::uint32_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)())){return 'm';}- if ((std::numeric_limits<std::int64_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)())+ if ((std::numeric_limits<std::int64_t>::min)() <= j.m_data.m_value.number_integer && j.m_data.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)()){return 'L';}@@ -16462,35 +16587,35 @@ class binary_writercase value_t::number_unsigned:{- if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int8_t>::max)()))+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int8_t>::max)())){return 'i';}- if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint8_t>::max)()))+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint8_t>::max)())){return 'U';}- if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int16_t>::max)()))+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int16_t>::max)())){return 'I';}- if (use_bjdata && j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint16_t>::max)()))+ if (use_bjdata && j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint16_t>::max)())){return 'u';}- if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())){return 'l';}- if (use_bjdata && j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint32_t>::max)()))+ if (use_bjdata && j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint32_t>::max)())){return 'm';}- if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))+ if (j.m_data.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)())){return 'L';}- if (use_bjdata && j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())+ if (use_bjdata && j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)()){return 'M';}@@ -16499,7 +16624,7 @@ class binary_writer}case value_t::number_float:- return get_ubjson_float_prefix(j.m_value.number_float);+ return get_ubjson_float_prefix(j.m_data.m_value.number_float);case value_t::string:return 'S';@@ -16548,7 +16673,7 @@ class binary_writerstd::size_t len = (value.at(key).empty() ? 0 : 1);for (const auto& el : value.at(key)){- len *= static_cast<std::size_t>(el.m_value.number_unsigned);+ len *= static_cast<std::size_t>(el.m_data.m_value.number_unsigned);}key = "_ArrayData_";@@ -16570,70 +16695,70 @@ class binary_writer{for (const auto& el : value.at(key)){- write_number(static_cast<std::uint8_t>(el.m_value.number_unsigned), true);+ write_number(static_cast<std::uint8_t>(el.m_data.m_value.number_unsigned), true);}}else if (dtype == 'i'){for (const auto& el : value.at(key)){- write_number(static_cast<std::int8_t>(el.m_value.number_integer), true);+ write_number(static_cast<std::int8_t>(el.m_data.m_value.number_integer), true);}}else if (dtype == 'u'){for (const auto& el : value.at(key)){- write_number(static_cast<std::uint16_t>(el.m_value.number_unsigned), true);+ write_number(static_cast<std::uint16_t>(el.m_data.m_value.number_unsigned), true);}}else if (dtype == 'I'){for (const auto& el : value.at(key)){- write_number(static_cast<std::int16_t>(el.m_value.number_integer), true);+ write_number(static_cast<std::int16_t>(el.m_data.m_value.number_integer), true);}}else if (dtype == 'm'){for (const auto& el : value.at(key)){- write_number(static_cast<std::uint32_t>(el.m_value.number_unsigned), true);+ write_number(static_cast<std::uint32_t>(el.m_data.m_value.number_unsigned), true);}}else if (dtype == 'l'){for (const auto& el : value.at(key)){- write_number(static_cast<std::int32_t>(el.m_value.number_integer), true);+ write_number(static_cast<std::int32_t>(el.m_data.m_value.number_integer), true);}}else if (dtype == 'M'){for (const auto& el : value.at(key)){- write_number(static_cast<std::uint64_t>(el.m_value.number_unsigned), true);+ write_number(static_cast<std::uint64_t>(el.m_data.m_value.number_unsigned), true);}}else if (dtype == 'L'){for (const auto& el : value.at(key)){- write_number(static_cast<std::int64_t>(el.m_value.number_integer), true);+ write_number(static_cast<std::int64_t>(el.m_data.m_value.number_integer), true);}}else if (dtype == 'd'){for (const auto& el : value.at(key)){- write_number(static_cast<float>(el.m_value.number_float), true);+ write_number(static_cast<float>(el.m_data.m_value.number_float), true);}}else if (dtype == 'D'){for (const auto& el : value.at(key)){- write_number(static_cast<double>(el.m_value.number_float), true);+ write_number(static_cast<double>(el.m_data.m_value.number_float), true);}}return false;@@ -16757,11 +16882,11 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/output/serializer.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//// SPDX-FileCopyrightText: 2008-2009 Björn Hoehrmann <[email protected]>-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -16782,11 +16907,11 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/detail/conversions/to_chars.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//// SPDX-FileCopyrightText: 2009 Florian Loitsch <https://florian.loitsch.com/>-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -17692,7 +17817,7 @@ void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)// NB: If the neighbors are computed for single-precision numbers, there is a single float// (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision// value is off by 1 ulp.-#if 0+#if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if)const boundaries w = compute_boundaries(static_cast<double>(value));#elseconst boundaries w = compute_boundaries(value);@@ -17994,11 +18119,11 @@ class serializerconst unsigned int indent_step,const unsigned int current_indent = 0){- switch (val.m_type)+ switch (val.m_data.m_type){case value_t::object:{- if (val.m_value.object->empty())+ if (val.m_data.m_value.object->empty()){o->write_characters("{}", 2);return;@@ -18016,8 +18141,8 @@ class serializer}// first n-1 elements- auto i = val.m_value.object->cbegin();- for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)+ auto i = val.m_data.m_value.object->cbegin();+ for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i){o->write_characters(indent_string.c_str(), new_indent);o->write_character('\"');@@ -18028,8 +18153,8 @@ class serializer}// last element- JSON_ASSERT(i != val.m_value.object->cend());- JSON_ASSERT(std::next(i) == val.m_value.object->cend());+ JSON_ASSERT(i != val.m_data.m_value.object->cend());+ JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend());o->write_characters(indent_string.c_str(), new_indent);o->write_character('\"');dump_escaped(i->first, ensure_ascii);@@ -18045,8 +18170,8 @@ class serializero->write_character('{');// first n-1 elements- auto i = val.m_value.object->cbegin();- for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)+ auto i = val.m_data.m_value.object->cbegin();+ for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i){o->write_character('\"');dump_escaped(i->first, ensure_ascii);@@ -18056,8 +18181,8 @@ class serializer}// last element- JSON_ASSERT(i != val.m_value.object->cend());- JSON_ASSERT(std::next(i) == val.m_value.object->cend());+ JSON_ASSERT(i != val.m_data.m_value.object->cend());+ JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend());o->write_character('\"');dump_escaped(i->first, ensure_ascii);o->write_characters("\":", 2);@@ -18071,7 +18196,7 @@ class serializercase value_t::array:{- if (val.m_value.array->empty())+ if (val.m_data.m_value.array->empty()){o->write_characters("[]", 2);return;@@ -18089,8 +18214,8 @@ class serializer}// first n-1 elements- for (auto i = val.m_value.array->cbegin();- i != val.m_value.array->cend() - 1; ++i)+ for (auto i = val.m_data.m_value.array->cbegin();+ i != val.m_data.m_value.array->cend() - 1; ++i){o->write_characters(indent_string.c_str(), new_indent);dump(*i, true, ensure_ascii, indent_step, new_indent);@@ -18098,9 +18223,9 @@ class serializer}// last element- JSON_ASSERT(!val.m_value.array->empty());+ JSON_ASSERT(!val.m_data.m_value.array->empty());o->write_characters(indent_string.c_str(), new_indent);- dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);+ dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);o->write_character('\n');o->write_characters(indent_string.c_str(), current_indent);@@ -18111,16 +18236,16 @@ class serializero->write_character('[');// first n-1 elements- for (auto i = val.m_value.array->cbegin();- i != val.m_value.array->cend() - 1; ++i)+ for (auto i = val.m_data.m_value.array->cbegin();+ i != val.m_data.m_value.array->cend() - 1; ++i){dump(*i, false, ensure_ascii, indent_step, current_indent);o->write_character(',');}// last element- JSON_ASSERT(!val.m_value.array->empty());- dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);+ JSON_ASSERT(!val.m_data.m_value.array->empty());+ dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);o->write_character(']');}@@ -18131,7 +18256,7 @@ class serializercase value_t::string:{o->write_character('\"');- dump_escaped(*val.m_value.string, ensure_ascii);+ dump_escaped(*val.m_data.m_value.string, ensure_ascii);o->write_character('\"');return;}@@ -18153,24 +18278,24 @@ class serializero->write_characters("\"bytes\": [", 10);- if (!val.m_value.binary->empty())+ if (!val.m_data.m_value.binary->empty()){- for (auto i = val.m_value.binary->cbegin();- i != val.m_value.binary->cend() - 1; ++i)+ for (auto i = val.m_data.m_value.binary->cbegin();+ i != val.m_data.m_value.binary->cend() - 1; ++i){dump_integer(*i);o->write_characters(", ", 2);}- dump_integer(val.m_value.binary->back());+ dump_integer(val.m_data.m_value.binary->back());}o->write_characters("],\n", 3);o->write_characters(indent_string.c_str(), new_indent);o->write_characters("\"subtype\": ", 11);- if (val.m_value.binary->has_subtype())+ if (val.m_data.m_value.binary->has_subtype()){- dump_integer(val.m_value.binary->subtype());+ dump_integer(val.m_data.m_value.binary->subtype());}else{@@ -18184,21 +18309,21 @@ class serializer{o->write_characters("{\"bytes\":[", 10);- if (!val.m_value.binary->empty())+ if (!val.m_data.m_value.binary->empty()){- for (auto i = val.m_value.binary->cbegin();- i != val.m_value.binary->cend() - 1; ++i)+ for (auto i = val.m_data.m_value.binary->cbegin();+ i != val.m_data.m_value.binary->cend() - 1; ++i){dump_integer(*i);o->write_character(',');}- dump_integer(val.m_value.binary->back());+ dump_integer(val.m_data.m_value.binary->back());}o->write_characters("],\"subtype\":", 12);- if (val.m_value.binary->has_subtype())+ if (val.m_data.m_value.binary->has_subtype()){- dump_integer(val.m_value.binary->subtype());+ dump_integer(val.m_data.m_value.binary->subtype());o->write_character('}');}else@@ -18211,7 +18336,7 @@ class serializercase value_t::boolean:{- if (val.m_value.boolean)+ if (val.m_data.m_value.boolean){o->write_characters("true", 4);}@@ -18224,19 +18349,19 @@ class serializercase value_t::number_integer:{- dump_integer(val.m_value.number_integer);+ dump_integer(val.m_data.m_value.number_integer);return;}case value_t::number_unsigned:{- dump_integer(val.m_value.number_unsigned);+ dump_integer(val.m_data.m_value.number_unsigned);return;}case value_t::number_float:{- dump_float(val.m_value.number_float);+ dump_float(val.m_data.m_value.number_float);return;}@@ -18810,8 +18935,8 @@ class serializer? (byte & 0x3fu) | (codep << 6u): (0xFFu >> type) & (byte);- std::size_t index = 256u + static_cast<size_t>(state) * 16u + static_cast<size_t>(type);- JSON_ASSERT(index < 400);+ const std::size_t index = 256u + static_cast<size_t>(state) * 16u + static_cast<size_t>(type);+ JSON_ASSERT(index < utf8d.size());state = utf8d[index];return state;}@@ -18878,10 +19003,10 @@ NLOHMANN_JSON_NAMESPACE_END// #include <nlohmann/ordered_map.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -18998,7 +19123,7 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,template<class KeyType, detail::enable_if_t<detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>- T & at(KeyType && key)+ T & at(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward){for (auto it = this->begin(); it != this->end(); ++it){@@ -19026,7 +19151,7 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,template<class KeyType, detail::enable_if_t<detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>- const T & at(KeyType && key) const+ const T & at(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward){for (auto it = this->begin(); it != this->end(); ++it){@@ -19060,7 +19185,7 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,template<class KeyType, detail::enable_if_t<detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>- size_type erase(KeyType && key)+ size_type erase(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward){for (auto it = this->begin(); it != this->end(); ++it){@@ -19151,7 +19276,7 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,template<class KeyType, detail::enable_if_t<detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>- size_type count(KeyType && key) const+ size_type count(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward){for (auto it = this->begin(); it != this->end(); ++it){@@ -19177,7 +19302,7 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,template<class KeyType, detail::enable_if_t<detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>- iterator find(KeyType && key)+ iterator find(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward){for (auto it = this->begin(); it != this->end(); ++it){@@ -19240,7 +19365,9 @@ NLOHMANN_JSON_NAMESPACE_END#if defined(JSON_HAS_CPP_17)- #include <any>+ #if JSON_HAS_STATIC_RTTI+ #include <any>+ #endif#include <string_view>#endif@@ -19271,6 +19398,7 @@ The invariants are checked by member function assert_invariant().*/NLOHMANN_BASIC_JSON_TPL_DECLARATIONclass basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions)+ : public ::nlohmann::detail::json_base_class<CustomBaseClass>{private:template<detail::value_t> friend struct detail::external_constructor;@@ -19297,6 +19425,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// workaround type for MSVCusing basic_json_t = NLOHMANN_BASIC_JSON_TPL;+ using json_base_class_t = ::nlohmann::detail::json_base_class<CustomBaseClass>;JSON_PRIVATE_UNLESS_TESTED:// convenience aliases for types residing in namespace detail;@@ -19368,7 +19497,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @}-/////////////////////// container types ///////////////////////@@ -19410,7 +19538,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @}-/// @brief returns the allocator associated with the container/// @sa https://json.nlohmann.me/api/basic_json/get_allocator/static allocator_type get_allocator()@@ -19425,7 +19552,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec{basic_json result;- result["copyright"] = "(C) 2013-2022 Niels Lohmann";+ result["copyright"] = "(C) 2013-2023 Niels Lohmann";result["name"] = "JSON for Modern C++";result["url"] = "https://github.com/nlohmann/json";result["version"]["string"] =@@ -19473,7 +19600,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specresult["compiler"] = {{"family", "unknown"}, {"version", "unknown"}};#endif-#if defined(_MSVC_LANG)result["compiler"]["c++"] = std::to_string(_MSVC_LANG);#elif defined(__cplusplus)@@ -19484,7 +19610,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specreturn result;}-///////////////////////////// JSON value data types /////////////////////////////@@ -19692,7 +19817,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specobject = nullptr; // silence warning, see #821if (JSON_HEDLEY_UNLIKELY(t == value_t::null)){- JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.11.2", nullptr)); // LCOV_EXCL_LINE+ JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.11.3", nullptr)); // LCOV_EXCL_LINE}break;}@@ -19731,6 +19856,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specvoid destroy(value_t t){+ if (+ (t == value_t::object && object == nullptr) ||+ (t == value_t::array && array == nullptr) ||+ (t == value_t::string && string == nullptr) ||+ (t == value_t::binary && binary == nullptr)+ )+ {+ //not initialized (e.g. due to exception in the ctor)+ return;+ }if (t == value_t::array || t == value_t::object){// flatten the current json_value to a heap-allocated stack@@ -19761,18 +19896,18 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// its children to the stack to be processed laterif (current_item.is_array()){- std::move(current_item.m_value.array->begin(), current_item.m_value.array->end(), std::back_inserter(stack));+ std::move(current_item.m_data.m_value.array->begin(), current_item.m_data.m_value.array->end(), std::back_inserter(stack));- current_item.m_value.array->clear();+ current_item.m_data.m_value.array->clear();}else if (current_item.is_object()){- for (auto&& it : *current_item.m_value.object)+ for (auto&& it : *current_item.m_data.m_value.object){stack.push_back(std::move(it.second));}- current_item.m_value.object->clear();+ current_item.m_data.m_value.object->clear();}// it's now safe that current_item get destructed@@ -19849,10 +19984,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec*/void assert_invariant(bool check_parents = true) const noexcept{- JSON_ASSERT(m_type != value_t::object || m_value.object != nullptr);- JSON_ASSERT(m_type != value_t::array || m_value.array != nullptr);- JSON_ASSERT(m_type != value_t::string || m_value.string != nullptr);- JSON_ASSERT(m_type != value_t::binary || m_value.binary != nullptr);+ JSON_ASSERT(m_data.m_type != value_t::object || m_data.m_value.object != nullptr);+ JSON_ASSERT(m_data.m_type != value_t::array || m_data.m_value.array != nullptr);+ JSON_ASSERT(m_data.m_type != value_t::string || m_data.m_value.string != nullptr);+ JSON_ASSERT(m_data.m_type != value_t::binary || m_data.m_value.binary != nullptr);#if JSON_DIAGNOSTICSJSON_TRY@@ -19871,11 +20006,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specvoid set_parents(){#if JSON_DIAGNOSTICS- switch (m_type)+ switch (m_data.m_type){case value_t::array:{- for (auto& element : *m_value.array)+ for (auto& element : *m_data.m_value.array){element.m_parent = this;}@@ -19884,7 +20019,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speccase value_t::object:{- for (auto& element : *m_value.object)+ for (auto& element : *m_data.m_value.object){element.second.m_parent = this;}@@ -19925,7 +20060,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec{// see https://github.com/nlohmann/json/issues/2838JSON_ASSERT(type() == value_t::array);- if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity))+ if (JSON_HEDLEY_UNLIKELY(m_data.m_value.array->capacity() != old_capacity)){// capacity has changed: update all parentsset_parents();@@ -19981,7 +20116,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief create an empty value with a given type/// @sa https://json.nlohmann.me/api/basic_json/basic_json/basic_json(const value_t v)- : m_type(v), m_value(v)+ : m_data(v){assert_invariant();}@@ -20055,12 +20190,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec*this = nullptr;break;case value_t::discarded:- m_type = value_t::discarded;+ m_data.m_type = value_t::discarded;break;default: // LCOV_EXCL_LINEJSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE}- JSON_ASSERT(m_type == val.type());+ JSON_ASSERT(m_data.m_type == val.type());set_parents();assert_invariant();}@@ -20076,7 +20211,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specbool is_an_object = std::all_of(init.begin(), init.end(),[](const detail::json_ref<basic_json>& element_ref){- return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[0].is_string();+ // The cast is to ensure op[size_type] is called, bearing in mind size_type may not be int;+ // (many string types can be constructed from 0 via its null-pointer guise, so we get a+ // broken call to op[key_type], the wrong semantics and a 4804 warning on Windows)+ return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[static_cast<size_type>(0)].is_string();});// adjust type if type deduction is not wanted@@ -20098,22 +20236,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (is_an_object){// the initializer list is a list of pairs -> create object- m_type = value_t::object;- m_value = value_t::object;+ m_data.m_type = value_t::object;+ m_data.m_value = value_t::object;for (auto& element_ref : init){auto element = element_ref.moved_or_copied();- m_value.object->emplace(- std::move(*((*element.m_value.array)[0].m_value.string)),- std::move((*element.m_value.array)[1]));+ m_data.m_value.object->emplace(+ std::move(*((*element.m_data.m_value.array)[0].m_data.m_value.string)),+ std::move((*element.m_data.m_value.array)[1]));}}else{// the initializer list describes an array -> create array- m_type = value_t::array;- m_value.array = create<array_t>(init.begin(), init.end());+ m_data.m_type = value_t::array;+ m_data.m_value.array = create<array_t>(init.begin(), init.end());}set_parents();@@ -20126,8 +20264,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specstatic basic_json binary(const typename binary_t::container_type& init){auto res = basic_json();- res.m_type = value_t::binary;- res.m_value = init;+ res.m_data.m_type = value_t::binary;+ res.m_data.m_value = init;return res;}@@ -20137,8 +20275,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specstatic basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype){auto res = basic_json();- res.m_type = value_t::binary;- res.m_value = binary_t(init, subtype);+ res.m_data.m_type = value_t::binary;+ res.m_data.m_value = binary_t(init, subtype);return res;}@@ -20148,8 +20286,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specstatic basic_json binary(typename binary_t::container_type&& init){auto res = basic_json();- res.m_type = value_t::binary;- res.m_value = std::move(init);+ res.m_data.m_type = value_t::binary;+ res.m_data.m_value = std::move(init);return res;}@@ -20159,8 +20297,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specstatic basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype){auto res = basic_json();- res.m_type = value_t::binary;- res.m_value = binary_t(std::move(init), subtype);+ res.m_data.m_type = value_t::binary;+ res.m_data.m_value = binary_t(std::move(init), subtype);return res;}@@ -20182,10 +20320,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief construct an array with count copies of given value/// @sa https://json.nlohmann.me/api/basic_json/basic_json/- basic_json(size_type cnt, const basic_json& val)- : m_type(value_t::array)+ basic_json(size_type cnt, const basic_json& val):+ m_data{cnt, val}{- m_value.array = create<array_t>(cnt, val);set_parents();assert_invariant();}@@ -20207,10 +20344,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec}// copy type from first iterator- m_type = first.m_object->m_type;+ m_data.m_type = first.m_object->m_data.m_type;// check if iterator range is complete for primitive values- switch (m_type)+ switch (m_data.m_type){case value_t::boolean:case value_t::number_float:@@ -20235,55 +20372,55 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specbreak;}- switch (m_type)+ switch (m_data.m_type){case value_t::number_integer:{- m_value.number_integer = first.m_object->m_value.number_integer;+ m_data.m_value.number_integer = first.m_object->m_data.m_value.number_integer;break;}case value_t::number_unsigned:{- m_value.number_unsigned = first.m_object->m_value.number_unsigned;+ m_data.m_value.number_unsigned = first.m_object->m_data.m_value.number_unsigned;break;}case value_t::number_float:{- m_value.number_float = first.m_object->m_value.number_float;+ m_data.m_value.number_float = first.m_object->m_data.m_value.number_float;break;}case value_t::boolean:{- m_value.boolean = first.m_object->m_value.boolean;+ m_data.m_value.boolean = first.m_object->m_data.m_value.boolean;break;}case value_t::string:{- m_value = *first.m_object->m_value.string;+ m_data.m_value = *first.m_object->m_data.m_value.string;break;}case value_t::object:{- m_value.object = create<object_t>(first.m_it.object_iterator,- last.m_it.object_iterator);+ m_data.m_value.object = create<object_t>(first.m_it.object_iterator,+ last.m_it.object_iterator);break;}case value_t::array:{- m_value.array = create<array_t>(first.m_it.array_iterator,- last.m_it.array_iterator);+ m_data.m_value.array = create<array_t>(first.m_it.array_iterator,+ last.m_it.array_iterator);break;}case value_t::binary:{- m_value = *first.m_object->m_value.binary;+ m_data.m_value = *first.m_object->m_data.m_value.binary;break;}@@ -20297,7 +20434,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specassert_invariant();}-///////////////////////////////////////// other constructors and destructor /////////////////////////////////////////@@ -20310,58 +20446,59 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief copy constructor/// @sa https://json.nlohmann.me/api/basic_json/basic_json/basic_json(const basic_json& other)- : m_type(other.m_type)+ : json_base_class_t(other){+ m_data.m_type = other.m_data.m_type;// check of passed value is validother.assert_invariant();- switch (m_type)+ switch (m_data.m_type){case value_t::object:{- m_value = *other.m_value.object;+ m_data.m_value = *other.m_data.m_value.object;break;}case value_t::array:{- m_value = *other.m_value.array;+ m_data.m_value = *other.m_data.m_value.array;break;}case value_t::string:{- m_value = *other.m_value.string;+ m_data.m_value = *other.m_data.m_value.string;break;}case value_t::boolean:{- m_value = other.m_value.boolean;+ m_data.m_value = other.m_data.m_value.boolean;break;}case value_t::number_integer:{- m_value = other.m_value.number_integer;+ m_data.m_value = other.m_data.m_value.number_integer;break;}case value_t::number_unsigned:{- m_value = other.m_value.number_unsigned;+ m_data.m_value = other.m_data.m_value.number_unsigned;break;}case value_t::number_float:{- m_value = other.m_value.number_float;+ m_data.m_value = other.m_data.m_value.number_float;break;}case value_t::binary:{- m_value = *other.m_value.binary;+ m_data.m_value = *other.m_data.m_value.binary;break;}@@ -20378,15 +20515,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief move constructor/// @sa https://json.nlohmann.me/api/basic_json/basic_json/basic_json(basic_json&& other) noexcept- : m_type(std::move(other.m_type)),- m_value(std::move(other.m_value))+ : json_base_class_t(std::forward<json_base_class_t>(other)),+ m_data(std::move(other.m_data)){// check that passed value is validother.assert_invariant(false);// invalidate payload- other.m_type = value_t::null;- other.m_value = {};+ other.m_data.m_type = value_t::null;+ other.m_data.m_value = {};set_parents();assert_invariant();@@ -20398,15 +20535,17 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specstd::is_nothrow_move_constructible<value_t>::value&&std::is_nothrow_move_assignable<value_t>::value&&std::is_nothrow_move_constructible<json_value>::value&&- std::is_nothrow_move_assignable<json_value>::value+ std::is_nothrow_move_assignable<json_value>::value&&+ std::is_nothrow_move_assignable<json_base_class_t>::value){// check that passed value is validother.assert_invariant();using std::swap;- swap(m_type, other.m_type);- swap(m_value, other.m_value);+ swap(m_data.m_type, other.m_data.m_type);+ swap(m_data.m_value, other.m_data.m_value);+ json_base_class_t::operator=(std::move(other));set_parents();assert_invariant();@@ -20418,7 +20557,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec~basic_json() noexcept{assert_invariant(false);- m_value.destroy(m_type);}/// @}@@ -20458,7 +20596,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @sa https://json.nlohmann.me/api/basic_json/type/constexpr value_t type() const noexcept{- return m_type;+ return m_data.m_type;}/// @brief return whether type is primitive@@ -20479,14 +20617,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @sa https://json.nlohmann.me/api/basic_json/is_null/constexpr bool is_null() const noexcept{- return m_type == value_t::null;+ return m_data.m_type == value_t::null;}/// @brief return whether value is a boolean/// @sa https://json.nlohmann.me/api/basic_json/is_boolean/constexpr bool is_boolean() const noexcept{- return m_type == value_t::boolean;+ return m_data.m_type == value_t::boolean;}/// @brief return whether value is a number@@ -20500,63 +20638,63 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/constexpr bool is_number_integer() const noexcept{- return m_type == value_t::number_integer || m_type == value_t::number_unsigned;+ return m_data.m_type == value_t::number_integer || m_data.m_type == value_t::number_unsigned;}/// @brief return whether value is an unsigned integer number/// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/constexpr bool is_number_unsigned() const noexcept{- return m_type == value_t::number_unsigned;+ return m_data.m_type == value_t::number_unsigned;}/// @brief return whether value is a floating-point number/// @sa https://json.nlohmann.me/api/basic_json/is_number_float/constexpr bool is_number_float() const noexcept{- return m_type == value_t::number_float;+ return m_data.m_type == value_t::number_float;}/// @brief return whether value is an object/// @sa https://json.nlohmann.me/api/basic_json/is_object/constexpr bool is_object() const noexcept{- return m_type == value_t::object;+ return m_data.m_type == value_t::object;}/// @brief return whether value is an array/// @sa https://json.nlohmann.me/api/basic_json/is_array/constexpr bool is_array() const noexcept{- return m_type == value_t::array;+ return m_data.m_type == value_t::array;}/// @brief return whether value is a string/// @sa https://json.nlohmann.me/api/basic_json/is_string/constexpr bool is_string() const noexcept{- return m_type == value_t::string;+ return m_data.m_type == value_t::string;}/// @brief return whether value is a binary array/// @sa https://json.nlohmann.me/api/basic_json/is_binary/constexpr bool is_binary() const noexcept{- return m_type == value_t::binary;+ return m_data.m_type == value_t::binary;}/// @brief return whether value is discarded/// @sa https://json.nlohmann.me/api/basic_json/is_discarded/constexpr bool is_discarded() const noexcept{- return m_type == value_t::discarded;+ return m_data.m_type == value_t::discarded;}/// @brief return the type of the JSON value (implicit)/// @sa https://json.nlohmann.me/api/basic_json/operator_value_t/constexpr operator value_t() const noexcept{- return m_type;+ return m_data.m_type;}/// @}@@ -20571,7 +20709,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec{if (JSON_HEDLEY_LIKELY(is_boolean())){- return m_value.boolean;+ return m_data.m_value.boolean;}JSON_THROW(type_error::create(302, detail::concat("type must be boolean, but is ", type_name()), this));@@ -20580,97 +20718,97 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// get a pointer to the value (object)object_t* get_impl_ptr(object_t* /*unused*/) noexcept{- return is_object() ? m_value.object : nullptr;+ return is_object() ? m_data.m_value.object : nullptr;}/// get a pointer to the value (object)constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept{- return is_object() ? m_value.object : nullptr;+ return is_object() ? m_data.m_value.object : nullptr;}/// get a pointer to the value (array)array_t* get_impl_ptr(array_t* /*unused*/) noexcept{- return is_array() ? m_value.array : nullptr;+ return is_array() ? m_data.m_value.array : nullptr;}/// get a pointer to the value (array)constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept{- return is_array() ? m_value.array : nullptr;+ return is_array() ? m_data.m_value.array : nullptr;}/// get a pointer to the value (string)string_t* get_impl_ptr(string_t* /*unused*/) noexcept{- return is_string() ? m_value.string : nullptr;+ return is_string() ? m_data.m_value.string : nullptr;}/// get a pointer to the value (string)constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept{- return is_string() ? m_value.string : nullptr;+ return is_string() ? m_data.m_value.string : nullptr;}/// get a pointer to the value (boolean)boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept{- return is_boolean() ? &m_value.boolean : nullptr;+ return is_boolean() ? &m_data.m_value.boolean : nullptr;}/// get a pointer to the value (boolean)constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept{- return is_boolean() ? &m_value.boolean : nullptr;+ return is_boolean() ? &m_data.m_value.boolean : nullptr;}/// get a pointer to the value (integer number)number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept{- return is_number_integer() ? &m_value.number_integer : nullptr;+ return is_number_integer() ? &m_data.m_value.number_integer : nullptr;}/// get a pointer to the value (integer number)constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept{- return is_number_integer() ? &m_value.number_integer : nullptr;+ return is_number_integer() ? &m_data.m_value.number_integer : nullptr;}/// get a pointer to the value (unsigned number)number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept{- return is_number_unsigned() ? &m_value.number_unsigned : nullptr;+ return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr;}/// get a pointer to the value (unsigned number)constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept{- return is_number_unsigned() ? &m_value.number_unsigned : nullptr;+ return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr;}/// get a pointer to the value (floating-point number)number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept{- return is_number_float() ? &m_value.number_float : nullptr;+ return is_number_float() ? &m_data.m_value.number_float : nullptr;}/// get a pointer to the value (floating-point number)constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept{- return is_number_float() ? &m_value.number_float : nullptr;+ return is_number_float() ? &m_data.m_value.number_float : nullptr;}/// get a pointer to the value (binary)binary_t* get_impl_ptr(binary_t* /*unused*/) noexcept{- return is_binary() ? m_value.binary : nullptr;+ return is_binary() ? m_data.m_value.binary : nullptr;}/// get a pointer to the value (binary)constexpr const binary_t* get_impl_ptr(const binary_t* /*unused*/) const noexcept{- return is_binary() ? m_value.binary : nullptr;+ return is_binary() ? m_data.m_value.binary : nullptr;}/*!@@ -21053,7 +21191,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER <= 1914))detail::negation<std::is_same<ValueType, std::string_view>>,#endif-#if defined(JSON_HAS_CPP_17)+#if defined(JSON_HAS_CPP_17) && JSON_HAS_STATIC_RTTIdetail::negation<std::is_same<ValueType, std::any>>,#endifdetail::is_detected_lazy<detail::get_template_function, const basic_json_t&, ValueType>@@ -21090,7 +21228,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @}-////////////////////// element access //////////////////////@@ -21108,7 +21245,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec{JSON_TRY{- return set_parent(m_value.array->at(idx));+ return set_parent(m_data.m_value.array->at(idx));}JSON_CATCH (std::out_of_range&){@@ -21131,7 +21268,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec{JSON_TRY{- return m_value.array->at(idx);+ return m_data.m_value.array->at(idx);}JSON_CATCH (std::out_of_range&){@@ -21155,8 +21292,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));}- auto it = m_value.object->find(key);- if (it == m_value.object->end())+ auto it = m_data.m_value.object->find(key);+ if (it == m_data.m_value.object->end()){JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this));}@@ -21175,8 +21312,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));}- auto it = m_value.object->find(std::forward<KeyType>(key));- if (it == m_value.object->end())+ auto it = m_data.m_value.object->find(std::forward<KeyType>(key));+ if (it == m_data.m_value.object->end()){JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward<KeyType>(key)), "' not found"), this));}@@ -21193,8 +21330,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));}- auto it = m_value.object->find(key);- if (it == m_value.object->end())+ auto it = m_data.m_value.object->find(key);+ if (it == m_data.m_value.object->end()){JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this));}@@ -21213,8 +21350,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this));}- auto it = m_value.object->find(std::forward<KeyType>(key));- if (it == m_value.object->end())+ auto it = m_data.m_value.object->find(std::forward<KeyType>(key));+ if (it == m_data.m_value.object->end()){JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward<KeyType>(key)), "' not found"), this));}@@ -21228,8 +21365,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// implicitly convert null value to an empty arrayif (is_null()){- m_type = value_t::array;- m_value.array = create<array_t>();+ m_data.m_type = value_t::array;+ m_data.m_value.array = create<array_t>();assert_invariant();}@@ -21237,17 +21374,17 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (JSON_HEDLEY_LIKELY(is_array())){// fill up array with null values if given idx is outside range- if (idx >= m_value.array->size())+ if (idx >= m_data.m_value.array->size()){#if JSON_DIAGNOSTICS// remember array size & capacity before resizing- const auto old_size = m_value.array->size();- const auto old_capacity = m_value.array->capacity();+ const auto old_size = m_data.m_value.array->size();+ const auto old_capacity = m_data.m_value.array->capacity();#endif- m_value.array->resize(idx + 1);+ m_data.m_value.array->resize(idx + 1);#if JSON_DIAGNOSTICS- if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity))+ if (JSON_HEDLEY_UNLIKELY(m_data.m_value.array->capacity() != old_capacity)){// capacity has changed: update all parentsset_parents();@@ -21261,7 +21398,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specassert_invariant();}- return m_value.array->operator[](idx);+ return m_data.m_value.array->operator[](idx);}JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a numeric argument with ", type_name()), this));@@ -21274,7 +21411,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// const operator[] only works for arraysif (JSON_HEDLEY_LIKELY(is_array())){- return m_value.array->operator[](idx);+ return m_data.m_value.array->operator[](idx);}JSON_THROW(type_error::create(305, detail::concat("cannot use operator[] with a numeric argument with ", type_name()), this));@@ -21287,15 +21424,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// implicitly convert null value to an empty objectif (is_null()){- m_type = value_t::object;- m_value.object = create<object_t>();+ m_data.m_type = value_t::object;+ m_data.m_value.object = create<object_t>();assert_invariant();}// operator[] only works for objectsif (JSON_HEDLEY_LIKELY(is_object())){- auto result = m_value.object->emplace(std::move(key), nullptr);+ auto result = m_data.m_value.object->emplace(std::move(key), nullptr);return set_parent(result.first->second);}@@ -21309,8 +21446,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// const operator[] only works for objectsif (JSON_HEDLEY_LIKELY(is_object())){- auto it = m_value.object->find(key);- JSON_ASSERT(it != m_value.object->end());+ auto it = m_data.m_value.object->find(key);+ JSON_ASSERT(it != m_data.m_value.object->end());return it->second;}@@ -21340,15 +21477,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// implicitly convert null value to an empty objectif (is_null()){- m_type = value_t::object;- m_value.object = create<object_t>();+ m_data.m_type = value_t::object;+ m_data.m_value.object = create<object_t>();assert_invariant();}// operator[] only works for objectsif (JSON_HEDLEY_LIKELY(is_object())){- auto result = m_value.object->emplace(std::forward<KeyType>(key), nullptr);+ auto result = m_data.m_value.object->emplace(std::forward<KeyType>(key), nullptr);return set_parent(result.first->second);}@@ -21364,8 +21501,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// const operator[] only works for objectsif (JSON_HEDLEY_LIKELY(is_object())){- auto it = m_value.object->find(std::forward<KeyType>(key));- JSON_ASSERT(it != m_value.object->end());+ auto it = m_data.m_value.object->find(std::forward<KeyType>(key));+ JSON_ASSERT(it != m_data.m_value.object->end());return it->second;}@@ -21602,7 +21739,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specIteratorType result = end();- switch (m_type)+ switch (m_data.m_type){case value_t::boolean:case value_t::number_float:@@ -21619,32 +21756,32 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (is_string()){AllocatorType<string_t> alloc;- std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);- std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);- m_value.string = nullptr;+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.string);+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.string, 1);+ m_data.m_value.string = nullptr;}else if (is_binary()){AllocatorType<binary_t> alloc;- std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.binary);- std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.binary, 1);- m_value.binary = nullptr;+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.binary);+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.binary, 1);+ m_data.m_value.binary = nullptr;}- m_type = value_t::null;+ m_data.m_type = value_t::null;assert_invariant();break;}case value_t::object:{- result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator);+ result.m_it.object_iterator = m_data.m_value.object->erase(pos.m_it.object_iterator);break;}case value_t::array:{- result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator);+ result.m_it.array_iterator = m_data.m_value.array->erase(pos.m_it.array_iterator);break;}@@ -21672,7 +21809,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specIteratorType result = end();- switch (m_type)+ switch (m_data.m_type){case value_t::boolean:case value_t::number_float:@@ -21690,33 +21827,33 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (is_string()){AllocatorType<string_t> alloc;- std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);- std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);- m_value.string = nullptr;+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.string);+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.string, 1);+ m_data.m_value.string = nullptr;}else if (is_binary()){AllocatorType<binary_t> alloc;- std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.binary);- std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.binary, 1);- m_value.binary = nullptr;+ std::allocator_traits<decltype(alloc)>::destroy(alloc, m_data.m_value.binary);+ std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_data.m_value.binary, 1);+ m_data.m_value.binary = nullptr;}- m_type = value_t::null;+ m_data.m_type = value_t::null;assert_invariant();break;}case value_t::object:{- result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator,+ result.m_it.object_iterator = m_data.m_value.object->erase(first.m_it.object_iterator,last.m_it.object_iterator);break;}case value_t::array:{- result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator,+ result.m_it.array_iterator = m_data.m_value.array->erase(first.m_it.array_iterator,last.m_it.array_iterator);break;}@@ -21741,7 +21878,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this));}- return m_value.object->erase(std::forward<KeyType>(key));+ return m_data.m_value.object->erase(std::forward<KeyType>(key));}template < typename KeyType, detail::enable_if_t <@@ -21754,10 +21891,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this));}- const auto it = m_value.object->find(std::forward<KeyType>(key));- if (it != m_value.object->end())+ const auto it = m_data.m_value.object->find(std::forward<KeyType>(key));+ if (it != m_data.m_value.object->end()){- m_value.object->erase(it);+ m_data.m_value.object->erase(it);return 1;}return 0;@@ -21795,7 +21932,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this));}- m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx));+ m_data.m_value.array->erase(m_data.m_value.array->begin() + static_cast<difference_type>(idx));}else{@@ -21805,7 +21942,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @}-////////////// lookup //////////////@@ -21821,7 +21957,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (is_object()){- result.m_it.object_iterator = m_value.object->find(key);+ result.m_it.object_iterator = m_data.m_value.object->find(key);}return result;@@ -21835,7 +21971,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (is_object()){- result.m_it.object_iterator = m_value.object->find(key);+ result.m_it.object_iterator = m_data.m_value.object->find(key);}return result;@@ -21851,7 +21987,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (is_object()){- result.m_it.object_iterator = m_value.object->find(std::forward<KeyType>(key));+ result.m_it.object_iterator = m_data.m_value.object->find(std::forward<KeyType>(key));}return result;@@ -21867,7 +22003,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (is_object()){- result.m_it.object_iterator = m_value.object->find(std::forward<KeyType>(key));+ result.m_it.object_iterator = m_data.m_value.object->find(std::forward<KeyType>(key));}return result;@@ -21878,7 +22014,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specsize_type count(const typename object_t::key_type& key) const{// return 0 for all nonobject types- return is_object() ? m_value.object->count(key) : 0;+ return is_object() ? m_data.m_value.object->count(key) : 0;}/// @brief returns the number of occurrences of a key in a JSON object@@ -21888,14 +22024,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specsize_type count(KeyType && key) const{// return 0 for all nonobject types- return is_object() ? m_value.object->count(std::forward<KeyType>(key)) : 0;+ return is_object() ? m_data.m_value.object->count(std::forward<KeyType>(key)) : 0;}/// @brief check the existence of an element in a JSON object/// @sa https://json.nlohmann.me/api/basic_json/contains/bool contains(const typename object_t::key_type& key) const{- return is_object() && m_value.object->find(key) != m_value.object->end();+ return is_object() && m_data.m_value.object->find(key) != m_data.m_value.object->end();}/// @brief check the existence of an element in a JSON object@@ -21904,7 +22040,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specdetail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>bool contains(KeyType && key) const{- return is_object() && m_value.object->find(std::forward<KeyType>(key)) != m_value.object->end();+ return is_object() && m_data.m_value.object->find(std::forward<KeyType>(key)) != m_data.m_value.object->end();}/// @brief check the existence of an element in a JSON object given a JSON pointer@@ -21923,7 +22059,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @}-///////////////// iterators /////////////////@@ -22062,7 +22197,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @}-//////////////// capacity ////////////////@@ -22074,7 +22208,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @sa https://json.nlohmann.me/api/basic_json/empty/bool empty() const noexcept{- switch (m_type)+ switch (m_data.m_type){case value_t::null:{@@ -22085,13 +22219,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speccase value_t::array:{// delegate call to array_t::empty()- return m_value.array->empty();+ return m_data.m_value.array->empty();}case value_t::object:{// delegate call to object_t::empty()- return m_value.object->empty();+ return m_data.m_value.object->empty();}case value_t::string:@@ -22113,7 +22247,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @sa https://json.nlohmann.me/api/basic_json/size/size_type size() const noexcept{- switch (m_type)+ switch (m_data.m_type){case value_t::null:{@@ -22124,13 +22258,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speccase value_t::array:{// delegate call to array_t::size()- return m_value.array->size();+ return m_data.m_value.array->size();}case value_t::object:{// delegate call to object_t::size()- return m_value.object->size();+ return m_data.m_value.object->size();}case value_t::string:@@ -22152,18 +22286,18 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @sa https://json.nlohmann.me/api/basic_json/max_size/size_type max_size() const noexcept{- switch (m_type)+ switch (m_data.m_type){case value_t::array:{// delegate call to array_t::max_size()- return m_value.array->max_size();+ return m_data.m_value.array->max_size();}case value_t::object:{// delegate call to object_t::max_size()- return m_value.object->max_size();+ return m_data.m_value.object->max_size();}case value_t::null:@@ -22184,7 +22318,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @}-///////////////// modifiers /////////////////@@ -22196,53 +22329,53 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @sa https://json.nlohmann.me/api/basic_json/clear/void clear() noexcept{- switch (m_type)+ switch (m_data.m_type){case value_t::number_integer:{- m_value.number_integer = 0;+ m_data.m_value.number_integer = 0;break;}case value_t::number_unsigned:{- m_value.number_unsigned = 0;+ m_data.m_value.number_unsigned = 0;break;}case value_t::number_float:{- m_value.number_float = 0.0;+ m_data.m_value.number_float = 0.0;break;}case value_t::boolean:{- m_value.boolean = false;+ m_data.m_value.boolean = false;break;}case value_t::string:{- m_value.string->clear();+ m_data.m_value.string->clear();break;}case value_t::binary:{- m_value.binary->clear();+ m_data.m_value.binary->clear();break;}case value_t::array:{- m_value.array->clear();+ m_data.m_value.array->clear();break;}case value_t::object:{- m_value.object->clear();+ m_data.m_value.object->clear();break;}@@ -22266,15 +22399,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// transform null object into an arrayif (is_null()){- m_type = value_t::array;- m_value = value_t::array;+ m_data.m_type = value_t::array;+ m_data.m_value = value_t::array;assert_invariant();}// add element to array (move semantics)- const auto old_capacity = m_value.array->capacity();- m_value.array->push_back(std::move(val));- set_parent(m_value.array->back(), old_capacity);+ const auto old_capacity = m_data.m_value.array->capacity();+ m_data.m_value.array->push_back(std::move(val));+ set_parent(m_data.m_value.array->back(), old_capacity);// if val is moved from, basic_json move constructor marks it null, so we do not call the destructor}@@ -22299,15 +22432,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// transform null object into an arrayif (is_null()){- m_type = value_t::array;- m_value = value_t::array;+ m_data.m_type = value_t::array;+ m_data.m_value = value_t::array;assert_invariant();}// add element to array- const auto old_capacity = m_value.array->capacity();- m_value.array->push_back(val);- set_parent(m_value.array->back(), old_capacity);+ const auto old_capacity = m_data.m_value.array->capacity();+ m_data.m_value.array->push_back(val);+ set_parent(m_data.m_value.array->back(), old_capacity);}/// @brief add an object to an array@@ -22331,13 +22464,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// transform null object into an objectif (is_null()){- m_type = value_t::object;- m_value = value_t::object;+ m_data.m_type = value_t::object;+ m_data.m_value = value_t::object;assert_invariant();}// add element to object- auto res = m_value.object->insert(val);+ auto res = m_data.m_value.object->insert(val);set_parent(res.first->second);}@@ -22387,15 +22520,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// transform null object into an arrayif (is_null()){- m_type = value_t::array;- m_value = value_t::array;+ m_data.m_type = value_t::array;+ m_data.m_value = value_t::array;assert_invariant();}// add element to array (perfect forwarding)- const auto old_capacity = m_value.array->capacity();- m_value.array->emplace_back(std::forward<Args>(args)...);- return set_parent(m_value.array->back(), old_capacity);+ const auto old_capacity = m_data.m_value.array->capacity();+ m_data.m_value.array->emplace_back(std::forward<Args>(args)...);+ return set_parent(m_data.m_value.array->back(), old_capacity);}/// @brief add an object to an object if key does not exist@@ -22412,13 +22545,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// transform null object into an objectif (is_null()){- m_type = value_t::object;- m_value = value_t::object;+ m_data.m_type = value_t::object;+ m_data.m_value = value_t::object;assert_invariant();}// add element to array (perfect forwarding)- auto res = m_value.object->emplace(std::forward<Args>(args)...);+ auto res = m_data.m_value.object->emplace(std::forward<Args>(args)...);set_parent(res.first->second);// create result iterator and set iterator to the result of emplace@@ -22436,14 +22569,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speciterator insert_iterator(const_iterator pos, Args&& ... args){iterator result(this);- JSON_ASSERT(m_value.array != nullptr);+ JSON_ASSERT(m_data.m_value.array != nullptr);- auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator);- m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);- result.m_it.array_iterator = m_value.array->begin() + insert_pos;+ auto insert_pos = std::distance(m_data.m_value.array->begin(), pos.m_it.array_iterator);+ m_data.m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);+ result.m_it.array_iterator = m_data.m_value.array->begin() + insert_pos;// This could have been written as:- // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);+ // result.m_it.array_iterator = m_data.m_value.array->insert(pos.m_it.array_iterator, cnt, val);// but the return value of insert is missing in GCC 4.8, so it is written this way instead.set_parents();@@ -22570,7 +22703,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", this));}- m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);+ m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);}/// @brief updates a JSON object from another object, overwriting existing keys@@ -22587,8 +22720,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// implicitly convert null value to an empty objectif (is_null()){- m_type = value_t::object;- m_value.object = create<object_t>();+ m_data.m_type = value_t::object;+ m_data.m_value.object = create<object_t>();assert_invariant();}@@ -22613,16 +22746,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec{if (merge_objects && it.value().is_object()){- auto it2 = m_value.object->find(it.key());- if (it2 != m_value.object->end())+ auto it2 = m_data.m_value.object->find(it.key());+ if (it2 != m_data.m_value.object->end()){it2->second.update(it.value(), true);continue;}}- m_value.object->operator[](it.key()) = it.value();+ m_data.m_value.object->operator[](it.key()) = it.value();#if JSON_DIAGNOSTICS- m_value.object->operator[](it.key()).m_parent = this;+ m_data.m_value.object->operator[](it.key()).m_parent = this;#endif}}@@ -22632,12 +22765,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specvoid swap(reference other) noexcept (std::is_nothrow_move_constructible<value_t>::value&&std::is_nothrow_move_assignable<value_t>::value&&- std::is_nothrow_move_constructible<json_value>::value&&+ std::is_nothrow_move_constructible<json_value>::value&& // NOLINT(cppcoreguidelines-noexcept-swap,performance-noexcept-swap)std::is_nothrow_move_assignable<json_value>::value){- std::swap(m_type, other.m_type);- std::swap(m_value, other.m_value);+ std::swap(m_data.m_type, other.m_data.m_type);+ std::swap(m_data.m_value, other.m_data.m_value);set_parents();other.set_parents();@@ -22649,7 +22782,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specfriend void swap(reference left, reference right) noexcept (std::is_nothrow_move_constructible<value_t>::value&&std::is_nothrow_move_assignable<value_t>::value&&- std::is_nothrow_move_constructible<json_value>::value&&+ std::is_nothrow_move_constructible<json_value>::value&& // NOLINT(cppcoreguidelines-noexcept-swap,performance-noexcept-swap)std::is_nothrow_move_assignable<json_value>::value){@@ -22658,13 +22791,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief exchanges the values/// @sa https://json.nlohmann.me/api/basic_json/swap/- void swap(array_t& other) // NOLINT(bugprone-exception-escape)+ void swap(array_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap){// swap only works for arraysif (JSON_HEDLEY_LIKELY(is_array())){using std::swap;- swap(*(m_value.array), other);+ swap(*(m_data.m_value.array), other);}else{@@ -22674,13 +22807,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief exchanges the values/// @sa https://json.nlohmann.me/api/basic_json/swap/- void swap(object_t& other) // NOLINT(bugprone-exception-escape)+ void swap(object_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap){// swap only works for objectsif (JSON_HEDLEY_LIKELY(is_object())){using std::swap;- swap(*(m_value.object), other);+ swap(*(m_data.m_value.object), other);}else{@@ -22690,13 +22823,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief exchanges the values/// @sa https://json.nlohmann.me/api/basic_json/swap/- void swap(string_t& other) // NOLINT(bugprone-exception-escape)+ void swap(string_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap){// swap only works for stringsif (JSON_HEDLEY_LIKELY(is_string())){using std::swap;- swap(*(m_value.string), other);+ swap(*(m_data.m_value.string), other);}else{@@ -22706,13 +22839,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec/// @brief exchanges the values/// @sa https://json.nlohmann.me/api/basic_json/swap/- void swap(binary_t& other) // NOLINT(bugprone-exception-escape)+ void swap(binary_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap){// swap only works for stringsif (JSON_HEDLEY_LIKELY(is_binary())){using std::swap;- swap(*(m_value.binary), other);+ swap(*(m_data.m_value.binary), other);}else{@@ -22728,7 +22861,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specif (JSON_HEDLEY_LIKELY(is_binary())){using std::swap;- swap(*(m_value.binary), other);+ swap(*(m_data.m_value.binary), other);}else{@@ -22756,31 +22889,31 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specswitch (lhs_type) \{ \case value_t::array: \- return (*lhs.m_value.array) op (*rhs.m_value.array); \+ return (*lhs.m_data.m_value.array) op (*rhs.m_data.m_value.array); \\case value_t::object: \- return (*lhs.m_value.object) op (*rhs.m_value.object); \+ return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \\case value_t::null: \return (null_result); \\case value_t::string: \- return (*lhs.m_value.string) op (*rhs.m_value.string); \+ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \\case value_t::boolean: \- return (lhs.m_value.boolean) op (rhs.m_value.boolean); \+ return (lhs.m_data.m_value.boolean) op (rhs.m_data.m_value.boolean); \\case value_t::number_integer: \- return (lhs.m_value.number_integer) op (rhs.m_value.number_integer); \+ return (lhs.m_data.m_value.number_integer) op (rhs.m_data.m_value.number_integer); \\case value_t::number_unsigned: \- return (lhs.m_value.number_unsigned) op (rhs.m_value.number_unsigned); \+ return (lhs.m_data.m_value.number_unsigned) op (rhs.m_data.m_value.number_unsigned); \\case value_t::number_float: \- return (lhs.m_value.number_float) op (rhs.m_value.number_float); \+ return (lhs.m_data.m_value.number_float) op (rhs.m_data.m_value.number_float); \\case value_t::binary: \- return (*lhs.m_value.binary) op (*rhs.m_value.binary); \+ return (*lhs.m_data.m_value.binary) op (*rhs.m_data.m_value.binary); \\case value_t::discarded: \default: \@@ -22789,27 +22922,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec} \else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) \{ \- return static_cast<number_float_t>(lhs.m_value.number_integer) op rhs.m_value.number_float; \+ return static_cast<number_float_t>(lhs.m_data.m_value.number_integer) op rhs.m_data.m_value.number_float; \} \else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) \{ \- return lhs.m_value.number_float op static_cast<number_float_t>(rhs.m_value.number_integer); \+ return lhs.m_data.m_value.number_float op static_cast<number_float_t>(rhs.m_data.m_value.number_integer); \} \else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) \{ \- return static_cast<number_float_t>(lhs.m_value.number_unsigned) op rhs.m_value.number_float; \+ return static_cast<number_float_t>(lhs.m_data.m_value.number_unsigned) op rhs.m_data.m_value.number_float; \} \else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) \{ \- return lhs.m_value.number_float op static_cast<number_float_t>(rhs.m_value.number_unsigned); \+ return lhs.m_data.m_value.number_float op static_cast<number_float_t>(rhs.m_data.m_value.number_unsigned); \} \else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) \{ \- return static_cast<number_integer_t>(lhs.m_value.number_unsigned) op rhs.m_value.number_integer; \+ return static_cast<number_integer_t>(lhs.m_data.m_value.number_unsigned) op rhs.m_data.m_value.number_integer; \} \else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) \{ \- return lhs.m_value.number_integer op static_cast<number_integer_t>(rhs.m_value.number_unsigned); \+ return lhs.m_data.m_value.number_integer op static_cast<number_integer_t>(rhs.m_data.m_value.number_unsigned); \} \else if(compares_unordered(lhs, rhs))\{\@@ -22826,8 +22959,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// an operation is computed as an odd number of inverses of othersstatic bool compares_unordered(const_reference lhs, const_reference rhs, bool inverse = false) noexcept{- if ((lhs.is_number_float() && std::isnan(lhs.m_value.number_float) && rhs.is_number())- || (rhs.is_number_float() && std::isnan(rhs.m_value.number_float) && lhs.is_number()))+ if ((lhs.is_number_float() && std::isnan(lhs.m_data.m_value.number_float) && rhs.is_number())+ || (rhs.is_number_float() && std::isnan(rhs.m_data.m_value.number_float) && lhs.is_number())){return true;}@@ -23171,7 +23304,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec#endif // JSON_NO_IO/// @}-/////////////////////// deserialization ///////////////////////@@ -23328,7 +23460,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specJSON_HEDLEY_RETURNS_NON_NULLconst char* type_name() const noexcept{- switch (m_type)+ switch (m_data.m_type){case value_t::null:return "null";@@ -23352,17 +23484,43 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec}}-JSON_PRIVATE_UNLESS_TESTED://////////////////////// member variables ////////////////////////- /// the type of the current element- value_t m_type = value_t::null;+ struct data+ {+ /// the type of the current element+ value_t m_type = value_t::null;++ /// the value of the current element+ json_value m_value = {};++ data(const value_t v)+ : m_type(v), m_value(v)+ {+ }++ data(size_type cnt, const basic_json& val)+ : m_type(value_t::array)+ {+ m_value.array = create<array_t>(cnt, val);+ }++ data() noexcept = default;+ data(data&&) noexcept = default;+ data(const data&) noexcept = delete;+ data& operator=(data&&) noexcept = delete;+ data& operator=(const data&) noexcept = delete;++ ~data() noexcept+ {+ m_value.destroy(m_type);+ }+ };- /// the value of the current element- json_value m_value = {};+ data m_data = {};#if JSON_DIAGNOSTICS/// a pointer to a parent value (for debugging purposes)@@ -23543,7 +23701,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specreturn from_cbor(ptr, ptr + len, strict, allow_exceptions, tag_handler);}-JSON_HEDLEY_WARN_UNUSED_RESULTJSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len))static basic_json from_cbor(detail::span_input_adapter&& i,@@ -23667,7 +23824,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specreturn res ? result : basic_json(value_t::discarded);}-/// @brief create a JSON value from an input in BJData format/// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/template<typename InputType>@@ -23890,7 +24046,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec}// make sure the top element of the pointer exists- json_pointer top_pointer = ptr.top();+ json_pointer const top_pointer = ptr.top();if (top_pointer != ptr){result.at(top_pointer);@@ -23902,7 +24058,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec// parent must exist when performing patch add per RFC6902 specsbasic_json& parent = result.at(ptr);- switch (parent.m_type)+ switch (parent.m_data.m_type){case value_t::null:case value_t::object:@@ -23948,7 +24104,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec};// wrapper for "remove" operation; remove value at ptr- const auto operation_remove = [this, &result](json_pointer & ptr)+ const auto operation_remove = [this, & result](json_pointer & ptr){// get reference to parent of JSON pointer ptrconst auto last_path = ptr.back();@@ -23991,13 +24147,13 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specbool string_type) -> basic_json &{// find value- auto it = val.m_value.object->find(member);+ auto it = val.m_data.m_value.object->find(member);// context-sensitive error message- const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\'');+ const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); // NOLINT(bugprone-unused-local-non-trivial-variable)// check if desired value is present- if (JSON_HEDLEY_UNLIKELY(it == val.m_value.object->end()))+ if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())){// NOLINTNEXTLINE(performance-inefficient-string-concatenation)JSON_THROW(parse_error::create(105, 0, detail::concat(error_msg, " must have member '", member, "'"), &val));@@ -24052,7 +24208,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specjson_pointer from_ptr(from_path);// the "from" location must exist - use at()- basic_json v = result.at(from_ptr);+ basic_json const v = result.at(from_ptr);// The move operation is functionally identical to a// "remove" operation on the "from" location, followed@@ -24069,7 +24225,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-specconst json_pointer from_ptr(from_path);// the "from" location must exist - use at()- basic_json v = result.at(from_ptr);+ basic_json const v = result.at(from_ptr);// The copy is functionally identical to an "add"// operation at the target location using the value@@ -24311,7 +24467,11 @@ inline namespace json_literals/// @brief user-defined string literal for JSON values/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/JSON_HEDLEY_NON_NULL(1)-inline nlohmann::json operator "" _json(const char* s, std::size_t n)+#if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0)+ inline nlohmann::json operator ""_json(const char* s, std::size_t n)+#else+ inline nlohmann::json operator "" _json(const char* s, std::size_t n)+#endif{return nlohmann::json::parse(s, s + n);}@@ -24319,7 +24479,11 @@ inline nlohmann::json operator "" _json(const char* s, std::size_t n)/// @brief user-defined string literal for JSON pointer/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/JSON_HEDLEY_NON_NULL(1)-inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n)+#if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0)+ inline nlohmann::json::json_pointer operator ""_json_pointer(const char* s, std::size_t n)+#else+ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n)+#endif{return nlohmann::json::json_pointer(std::string(s, n));}@@ -24338,7 +24502,7 @@ namespace std // NOLINT(cert-dcl58-cpp)/// @brief hash value for JSON objects/// @sa https://json.nlohmann.me/api/basic_json/std_hash/NLOHMANN_BASIC_JSON_TPL_DECLARATION-struct hash<nlohmann::NLOHMANN_BASIC_JSON_TPL>+struct hash<nlohmann::NLOHMANN_BASIC_JSON_TPL> // NOLINT(cert-dcl58-cpp){std::size_t operator()(const nlohmann::NLOHMANN_BASIC_JSON_TPL& j) const{@@ -24371,8 +24535,8 @@ struct less< ::nlohmann::detail::value_t> // do not remove the space after '<',/// @brief exchanges the values of two JSON objects/// @sa https://json.nlohmann.me/api/basic_json/std_swap/NLOHMANN_BASIC_JSON_TPL_DECLARATION-inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name)- is_nothrow_move_constructible<nlohmann::NLOHMANN_BASIC_JSON_TPL>::value&& // NOLINT(misc-redundant-expression)+inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name, cert-dcl58-cpp)+ is_nothrow_move_constructible<nlohmann::NLOHMANN_BASIC_JSON_TPL>::value&& // NOLINT(misc-redundant-expression,cppcoreguidelines-noexcept-swap,performance-noexcept-swap)is_nothrow_move_assignable<nlohmann::NLOHMANN_BASIC_JSON_TPL>::value){j1.swap(j2);@@ -24383,17 +24547,22 @@ inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC} // namespace std#if JSON_USE_GLOBAL_UDLS- using nlohmann::literals::json_literals::operator "" _json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers)- using nlohmann::literals::json_literals::operator "" _json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers)+ #if !defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0)+ using nlohmann::literals::json_literals::operator ""_json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers)+ using nlohmann::literals::json_literals::operator ""_json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers)+ #else+ using nlohmann::literals::json_literals::operator "" _json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers)+ using nlohmann::literals::json_literals::operator "" _json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers)+ #endif#endif// #include <nlohmann/detail/macro_unscope.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MIT@@ -24428,16 +24597,17 @@ inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC#undef JSON_HAS_EXPERIMENTAL_FILESYSTEM#undef JSON_HAS_THREE_WAY_COMPARISON#undef JSON_HAS_RANGES+ #undef JSON_HAS_STATIC_RTTI#undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON#endif// #include <nlohmann/thirdparty/hedley/hedley_undef.hpp>// __ _____ _____ _____// __| | __| | | | JSON for Modern C++-// | | |__ | | | | | | version 3.11.2+// | | |__ | | | | | | version 3.11.3// |_____|_____|_____|_|___| https://github.com/nlohmann/json//-// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>// SPDX-License-Identifier: MITdiff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 658963ef..ba3f6f75 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -166,10 +166,13 @@ enum llm_arch {LLM_ARCH_INTERNLM2,LLM_ARCH_MINICPM,LLM_ARCH_GEMMA,+ LLM_ARCH_STARCODER2,+ LLM_ARCH_MAMBA,+ LLM_ARCH_COMMAND_R,LLM_ARCH_UNKNOWN,};-static std::map<llm_arch, const char *> LLM_ARCH_NAMES = {+static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_LLAMA, "llama" },{ LLM_ARCH_FALCON, "falcon" },{ LLM_ARCH_GPT2, "gpt2" },@@ -193,6 +196,10 @@ static std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_INTERNLM2, "internlm2" },{ LLM_ARCH_MINICPM, "minicpm" },{ LLM_ARCH_GEMMA, "gemma" },+ { LLM_ARCH_STARCODER2, "starcoder2" },+ { LLM_ARCH_MAMBA, "mamba" },+ { LLM_ARCH_COMMAND_R, "command-r" },+ { LLM_ARCH_UNKNOWN, "(unknown)" },};enum llm_kv {@@ -207,6 +214,7 @@ enum llm_kv {LLM_KV_GENERAL_SOURCE_URL,LLM_KV_GENERAL_SOURCE_HF_REPO,+ LLM_KV_VOCAB_SIZE,LLM_KV_CONTEXT_LENGTH,LLM_KV_EMBEDDING_LENGTH,LLM_KV_BLOCK_COUNT,@@ -216,6 +224,7 @@ enum llm_kv {LLM_KV_EXPERT_COUNT,LLM_KV_EXPERT_USED_COUNT,LLM_KV_POOLING_TYPE,+ LLM_KV_LOGIT_SCALE,LLM_KV_ATTENTION_HEAD_COUNT,LLM_KV_ATTENTION_HEAD_COUNT_KV,@@ -235,6 +244,11 @@ enum llm_kv {LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,LLM_KV_ROPE_SCALING_FINETUNED,+ LLM_KV_SSM_INNER_SIZE,+ LLM_KV_SSM_CONV_KERNEL,+ LLM_KV_SSM_STATE_SIZE,+ LLM_KV_SSM_TIME_STEP_RANK,+LLM_KV_TOKENIZER_MODEL,LLM_KV_TOKENIZER_LIST,LLM_KV_TOKENIZER_TOKEN_TYPE,@@ -253,7 +267,7 @@ enum llm_kv {LLM_KV_TOKENIZER_RWKV,};-static std::map<llm_kv, const char *> LLM_KV_NAMES = {+static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" },{ LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" },{ LLM_KV_GENERAL_ALIGNMENT, "general.alignment" },@@ -265,6 +279,7 @@ static std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_GENERAL_SOURCE_URL, "general.source.url" },{ LLM_KV_GENERAL_SOURCE_HF_REPO, "general.source.huggingface.repository" },+ { LLM_KV_VOCAB_SIZE, "%s.vocab_size" },{ LLM_KV_CONTEXT_LENGTH, "%s.context_length" },{ LLM_KV_EMBEDDING_LENGTH, "%s.embedding_length" },{ LLM_KV_BLOCK_COUNT, "%s.block_count" },@@ -274,6 +289,7 @@ static std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_EXPERT_COUNT, "%s.expert_count" },{ LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" },{ LLM_KV_POOLING_TYPE , "%s.pooling_type" },+ { LLM_KV_LOGIT_SCALE, "%s.logit_scale" },{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },@@ -293,6 +309,11 @@ static std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" },{ LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" },+ { LLM_KV_SSM_CONV_KERNEL, "%s.ssm.conv_kernel" },+ { LLM_KV_SSM_INNER_SIZE, "%s.ssm.inner_size" },+ { LLM_KV_SSM_STATE_SIZE, "%s.ssm.state_size" },+ { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },+{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },@@ -317,7 +338,7 @@ struct LLM_KV {llm_arch arch;std::string operator()(llm_kv kv) const {- return ::format(LLM_KV_NAMES[kv], LLM_ARCH_NAMES[arch]);+ return ::format(LLM_KV_NAMES.at(kv), LLM_ARCH_NAMES.at(arch));}};@@ -350,9 +371,16 @@ enum llm_tensor {LLM_TENSOR_ATTN_Q_NORM,LLM_TENSOR_ATTN_K_NORM,LLM_TENSOR_LAYER_OUT_NORM,+ LLM_TENSOR_SSM_IN,+ LLM_TENSOR_SSM_CONV1D,+ LLM_TENSOR_SSM_X,+ LLM_TENSOR_SSM_DT,+ LLM_TENSOR_SSM_A,+ LLM_TENSOR_SSM_D,+ LLM_TENSOR_SSM_OUT,};-static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {+static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {{LLM_ARCH_LLAMA,{@@ -466,6 +494,7 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output"},{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },@@ -734,6 +763,55 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_STARCODER2,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },+ {+ LLM_ARCH_MAMBA,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_SSM_IN, "blk.%d.ssm_in" },+ { LLM_TENSOR_SSM_CONV1D, "blk.%d.ssm_conv1d" },+ { LLM_TENSOR_SSM_X, "blk.%d.ssm_x" },+ { LLM_TENSOR_SSM_DT, "blk.%d.ssm_dt" },+ { LLM_TENSOR_SSM_A, "blk.%d.ssm_a" },+ { LLM_TENSOR_SSM_D, "blk.%d.ssm_d" },+ { LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" },+ },+ },+ {+ LLM_ARCH_COMMAND_R,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_UNKNOWN,{@@ -767,38 +845,38 @@ struct LLM_TN {llm_arch arch;std::string operator()(llm_tensor tensor) const {- if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {return "__missing__";}- return LLM_TENSOR_NAMES[arch].at(tensor);+ return LLM_TENSOR_NAMES.at(arch).at(tensor);}std::string operator()(llm_tensor tensor, const std::string & suffix) const {- if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {return "__missing__";}- return LLM_TENSOR_NAMES[arch].at(tensor) + "." + suffix;+ return LLM_TENSOR_NAMES.at(arch).at(tensor) + "." + suffix;}std::string operator()(llm_tensor tensor, int bid) const {- if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {return "__missing__";}- return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid);+ return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid);}std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const {- if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {return "__missing__";}- return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix;+ return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid) + "." + suffix;}std::string operator()(llm_tensor tensor, const std::string & suffix, int bid, int xid) const {- if (LLM_TENSOR_NAMES[arch].find(tensor) == LLM_TENSOR_NAMES[arch].end()) {+ if (LLM_TENSOR_NAMES.at(arch).find(tensor) == LLM_TENSOR_NAMES.at(arch).end()) {return "__missing__";}- return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid, xid) + "." + suffix;+ return ::format(LLM_TENSOR_NAMES.at(arch).at(tensor).c_str(), bid, xid) + "." + suffix;}};@@ -806,16 +884,16 @@ struct LLM_TN {// gguf helpers//-static std::map<int32_t, const char *> LLAMA_ROPE_SCALING_TYPES = {+static const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {{ LLAMA_ROPE_SCALING_TYPE_NONE, "none" },{ LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" },{ LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" },};-static int32_t llama_rope_scaling_type_from_string(const std::string & name) {+static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {if (kv.second == name) {- return kv.first;+ return (llama_rope_scaling_type) kv.first;}}@@ -877,23 +955,22 @@ static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {}//-// ggml helpers+// llama helpers//-static void ggml_graph_compute_helper(std::vector<uint8_t> & buf, ggml_cgraph * graph, int n_threads) {- struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);-- if (plan.work_size > 0) {- buf.resize(plan.work_size);- plan.work_data = buf.data();+#if defined(_WIN32)+static std::string llama_format_win_err(DWORD err) {+ LPSTR buf;+ size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,+ NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);+ if (!size) {+ return "FormatMessageA failed";}-- ggml_graph_compute(graph, &plan);+ std::string ret(buf, size);+ LocalFree(buf);+ return ret;}--//-// llama helpers-//+#endiftemplate <typename T>struct no_init {@@ -1339,6 +1416,7 @@ enum e_model {MODEL_20B,MODEL_30B,MODEL_34B,+ MODEL_35B,MODEL_40B,MODEL_65B,MODEL_70B,@@ -1376,16 +1454,23 @@ struct llama_hparams {float rope_freq_base_train;float rope_freq_scale_train;uint32_t n_yarn_orig_ctx;- int32_t rope_scaling_type_train;++ // for State Space Models+ uint32_t ssm_d_conv = 0;+ uint32_t ssm_d_inner = 0;+ uint32_t ssm_d_state = 0;+ uint32_t ssm_dt_rank = 0;float f_clamp_kqv = 0.0f;float f_max_alibi_bias = 0.0f;+ float f_logit_scale = 0.0f;bool causal_attn = true;bool need_kq_pos = false;- enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE;- enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE;+ enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE;+ enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE;+ enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE;bool operator!=(const llama_hparams & other) const {if (this->vocab_only != other.vocab_only) return true;@@ -1405,6 +1490,11 @@ struct llama_hparams {if (this->rope_finetuned != other.rope_finetuned) return true;if (this->n_yarn_orig_ctx != other.n_yarn_orig_ctx) return true;+ if (this->ssm_d_conv != other.ssm_d_conv) return true;+ if (this->ssm_d_inner != other.ssm_d_inner) return true;+ if (this->ssm_d_state != other.ssm_d_state) return true;+ if (this->ssm_dt_rank != other.ssm_dt_rank) return true;+const float EPSILON = 1e-9f;if (!is_float_close(this->f_norm_eps, other.f_norm_eps, EPSILON)) return true;@@ -1416,6 +1506,9 @@ struct llama_hparams {}uint32_t n_gqa() const {+ if (n_head_kv == 0) {+ return 0;+ }return n_head/n_head_kv;}@@ -1426,16 +1519,29 @@ struct llama_hparams {uint32_t n_embd_v_gqa() const { // dimension of value embeddings across all k-v headsreturn n_embd_head_v * n_head_kv;}++ uint32_t n_embd_k_s() const { // dimension of the rolling state embeddings+ // corresponds to Mamba's conv_states size+ // TODO: maybe support other convolution strides than 1+ // NOTE: since the first column of the conv_state is shifted out each time, it's not actually needed+ return (ssm_d_conv > 0 ? ssm_d_conv - 1 : 0) * ssm_d_inner;+ }++ uint32_t n_embd_v_s() const { // dimension of the recurrent state embeddings+ // corresponds to Mamba's ssm_states size+ return ssm_d_state * ssm_d_inner;+ }};struct llama_cparams {- uint32_t n_ctx; // context size used during inference+ uint32_t n_ctx; // context size used during inferenceuint32_t n_batch;+ uint32_t n_ubatch;uint32_t n_threads; // number of threads to use for generationuint32_t n_threads_batch; // number of threads to use for batch processing- float rope_freq_base;- float rope_freq_scale;+ float rope_freq_base;+ float rope_freq_scale;uint32_t n_yarn_orig_ctx;// These hyperparameters are not exposed in GGUF, because all@@ -1446,9 +1552,11 @@ struct llama_cparams {float yarn_beta_slow;float defrag_thold;- bool mul_mat_q;+ bool embeddings;+ bool causal_attn;bool offload_kqv;- bool do_pooling;++ enum llama_pooling_type pooling_type;ggml_backend_sched_eval_callback cb_eval;void * cb_eval_user_data;@@ -1502,11 +1610,27 @@ struct llama_layer {struct ggml_tensor * ffn_down_b; // b2struct ggml_tensor * ffn_up_b; // b3struct ggml_tensor * ffn_act;++ // mamba proj+ struct ggml_tensor * ssm_in;+ struct ggml_tensor * ssm_x;+ struct ggml_tensor * ssm_dt;+ struct ggml_tensor * ssm_out;++ // mamba+ struct ggml_tensor * ssm_conv1d;+ struct ggml_tensor * ssm_a;+ struct ggml_tensor * ssm_d;++ // mamba bias+ struct ggml_tensor * ssm_conv1d_b;+ struct ggml_tensor * ssm_dt_b;};struct llama_kv_cell {llama_pos pos = -1;llama_pos delta = 0;+ int32_t src = 0; // used by recurrent state models to copy statesstd::set<llama_seq_id> seq_id;@@ -1527,6 +1651,9 @@ struct llama_kv_cell {struct llama_kv_cache {bool has_shift = false;bool do_defrag = false;+ bool do_copy = false;+ // with recurrent state models, a cell can hold the state for more than one past token+ bool recurrent = false;// Note: The value of head isn't only used to optimize searching// for a free KV slot. llama_decode_internal also uses it, so it@@ -1567,6 +1694,31 @@ struct llama_kv_cache {}};+struct llama_control_vector {+ std::vector<struct ggml_tensor *> tensors; // per layer+ std::vector<struct ggml_context *> ctxs;+ std::vector<ggml_backend_buffer_t> bufs;++ int32_t layer_start = -1;+ int32_t layer_end = -1;++ ggml_tensor * tensor_for(int il) const {+ if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) {+ return nullptr;+ }+ return tensors[il];+ }++ ~llama_control_vector() {+ for (struct ggml_context * ctx : ctxs) {+ ggml_free(ctx);+ }+ for (ggml_backend_buffer_t buf : bufs) {+ ggml_backend_buffer_free(buf);+ }+ }+};+struct llama_vocab {using id = int32_t;using token = std::string;@@ -1688,6 +1840,11 @@ struct llama_model {ggml_free(ctx);}for (ggml_backend_buffer_t buf : bufs) {+// #ifdef GGML_USE_CUBLAS+ if (ggml_backend_buffer_get_type(buf) == ggml_backend_cpu_buffer_type()) {+ ggml_backend_cuda_unregister_host_buffer(ggml_backend_buffer_get_base(buf));+ }+// #endifggml_backend_buffer_free(buf);}}@@ -1706,14 +1863,15 @@ struct llama_context {ggml_vk_free_cpu_assist();#endif- ggml_backend_buffer_free(buf_input);- ggml_free(ctx_input);+ ggml_backend_buffer_free(buf_output);}llama_cparams cparams;std::vector<ggml_backend_t> backends;+// #ifdef GGML_USE_METALggml_backend_t backend_metal = nullptr;+// #endifggml_backend_t backend_cpu = nullptr;const llama_model & model;@@ -1731,36 +1889,57 @@ struct llama_context {int64_t t_p_eval_us = 0;int64_t t_eval_us = 0;+ int64_t t_compute_start_us = 0;+ int64_t n_queued_tokens = 0;+int32_t n_sample = 0; // number of tokens sampledint32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1)int32_t n_eval = 0; // number of eval calls+ // host buffer for the model output (logits and embeddings)+ ggml_backend_buffer_t buf_output = nullptr;+// decode output (2-dimensional array: [n_tokens][n_vocab])- std::vector<float> logits;+ size_t logits_size = 0;+ float * logits = nullptr;+#ifndef NDEBUG// guard against access to unset logitsstd::vector<bool> logits_valid;#endifbool logits_all = false;- // input embedding (1-dimensional array: [n_embd])- std::vector<float> embedding;+ // embeddings output (2-dimensional array: [n_tokens][n_embd])+ // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE+ size_t embd_size = 0;+ float * embd = nullptr;++ // sequence embeddings output (map of [n_embd] vectors)+ // populated only when pooling_type != LLAMA_POOLING_TYPE_NONE+ std::map<llama_seq_id, std::vector<float>> embd_seq;// memory buffers used to evaluate the modelstd::vector<uint8_t> buf_compute_meta;ggml_backend_sched_t sched = nullptr;+ ggml_abort_callback abort_callback = nullptr;+ void * abort_callback_data = nullptr;+// input tensors- ggml_backend_buffer_t buf_input = nullptr;- ggml_context * ctx_input = nullptr;struct ggml_tensor * inp_tokens; // I32 [n_batch]struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch]struct ggml_tensor * inp_pos; // I32 [n_batch]- struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch]- struct ggml_tensor * inp_KQ_pos; // F32 [n_ctx]- struct ggml_tensor * inp_K_shift; // I32 [n_ctx]+ struct ggml_tensor * inp_KQ_mask; // F32 [kv_size, n_batch]+ struct ggml_tensor * inp_KQ_pos; // F32 [kv_size]+ struct ggml_tensor * inp_K_shift; // I32 [kv_size]struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch]struct ggml_tensor * inp_cls; // I32 [n_batch]+ struct ggml_tensor * inp_s_copy; // I32 [kv_size]+ struct ggml_tensor * inp_s_mask; // F32 [1, kv_size]+ struct ggml_tensor * inp_s_seq; // I32 [kv_size, n_batch]++ // control vectors+ struct llama_control_vector cvec;#ifdef GGML_USE_MPIggml_mpi_context * ctx_mpi = NULL;@@ -1776,25 +1955,42 @@ static bool llama_kv_cache_init(const llama_model & model,ggml_type type_k,ggml_type type_v,- uint32_t n_ctx,+ uint32_t kv_size,bool offload) {const struct llama_hparams & hparams = model.hparams;- const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa();- const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa();+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s();+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa() + hparams.n_embd_v_s();const int64_t n_layer = hparams.n_layer;cache.has_shift = false;+ // TODO: find a nicer way to add other recurrent model architectures+ cache.recurrent = model.arch == LLM_ARCH_MAMBA;++ // TODO: support mixed reccurent Transformer architectues+ // NOTE: (!a || b) is a logical implication (a -> b)+ GGML_ASSERT(!cache.recurrent || n_embd_k_gqa == hparams.n_embd_k_s());+ GGML_ASSERT(!cache.recurrent || n_embd_v_gqa == hparams.n_embd_v_s());+ GGML_ASSERT( cache.recurrent || n_embd_k_gqa == hparams.n_embd_k_gqa());+ GGML_ASSERT( cache.recurrent || n_embd_v_gqa == hparams.n_embd_v_gqa());+cache.head = 0;- cache.size = n_ctx;+ cache.size = kv_size;cache.used = 0;cache.type_k = type_k;cache.type_v = type_v;cache.cells.clear();- cache.cells.resize(n_ctx);+ cache.cells.resize(kv_size);++ if (cache.recurrent) {+ // init state copy sources+ for (uint32_t i = 0; i < cache.size; ++i) {+ cache.cells[i].src = i;+ }+ }#ifdef GGML_USE_CLBLASToffload = false;@@ -1833,8 +2029,8 @@ static bool llama_kv_cache_init(for (int i = 0; i < (int) n_layer; i++) {struct ggml_context * ctx = offload ? ctx_map.at(model.buft_layer[i].buft) : cache.ctxs.front();- ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*n_ctx);- ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*n_ctx);+ ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*kv_size);+ ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*kv_size);ggml_format_name(k, "cache_k_l%d", i);ggml_format_name(v, "cache_v_l%d", i);cache.k_l.push_back(k);@@ -1868,6 +2064,54 @@ static bool llama_kv_cache_find_slot(const uint32_t n_ctx = cache.size;const uint32_t n_tokens = batch.n_tokens;+ if (cache.recurrent) {+ // For recurrent state architectures (like Mamba),+ // each KV cache cell can store the state for a whole sequence.++ llama_seq_id min = cache.size - 1;+ llama_seq_id max = 0;++ for (uint32_t i = 0; i < n_tokens; ++i) {+ for (int32_t j = 0; j < batch.n_seq_id[i]; ++j) {+ llama_seq_id seq_id = batch.seq_id[i][j];+ // make sure it's a valid seq_id+ if ((uint32_t) seq_id < cache.size) {+ if (seq_id > max) {+ max = seq_id;+ }+ if (seq_id < min) {+ min = seq_id;+ }+ // Assuming the tokens are in-order+ if (batch.pos[i] != cache.cells[seq_id].pos + 1) {+ // What should happen when the pos backtracks or skips a value?+ // Clearing the state mid-batch would require special-casing which isn't done.+ LLAMA_LOG_WARN("%s: non-consecutive token position %d after %d for sequence %d\n",+ __func__, batch.pos[i], cache.cells[seq_id].pos, seq_id);+ }+ if (cache.cells[seq_id].pos < 0 && 0 <= batch.pos[i]) {+ cache.used += 1;+ }+ cache.cells[seq_id].pos = batch.pos[i];+ // NOTE: seq_ids are not inserted here; they are handled when the input tensors are set+ } else {+ // too big seq_id+ // TODO: would it be possible to resize the KV cache size instead?+ LLAMA_LOG_ERROR("%s: seq_id=%d >= kv_size=%d Try using a bigger --parallel value\n", __func__, seq_id, cache.size);+ return false;+ }+ }+ }++ // allow getting the range of used cells, from head to head + n+ cache.head = min;+ cache.n = max - min + 1;++ // sanity check+ return max >= min;+ }+ // otherwise, one cell per token.+if (n_tokens > n_ctx) {LLAMA_LOG_ERROR("%s: n_tokens=%d > n_ctx=%d\n", __func__, n_tokens, n_ctx);return false;@@ -1916,10 +2160,12 @@ static bool llama_kv_cache_find_slot(}// find how many cells are currently in use-static int32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) {- for (uint32_t i = cache.size - 1; i > 0; --i) {- if (cache.cells[i].pos >= 0 && !cache.cells[i].is_empty()) {- return i + 1;+static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) {+ for (uint32_t i = cache.size; i > 0; --i) {+ const llama_kv_cell & cell = cache.cells[i - 1];++ if (cell.pos >= 0 && !cell.is_empty()) {+ return i;}}@@ -1935,7 +2181,7 @@ static void llama_kv_cache_clear(struct llama_kv_cache & cache) {cache.used = 0;}-static void llama_kv_cache_seq_rm(+static bool llama_kv_cache_seq_rm(struct llama_kv_cache & cache,llama_seq_id seq_id,llama_pos p0,@@ -1945,6 +2191,25 @@ static void llama_kv_cache_seq_rm(if (p0 < 0) p0 = 0;if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();+ // models like Mamba can't have a state partially erased+ if (cache.recurrent) {+ if (seq_id >= (int64_t) cache.size) {+ // could be fatal+ return false;+ }+ if (0 <= seq_id) {+ // partial intersection is invalid+ if ((0 < p0 && p0 <= cache.cells[seq_id].pos) || (0 < p1 && p1 <= cache.cells[seq_id].pos)) {+ return false;+ }+ } else {+ // seq_id is negative, then the range should include everything or nothing+ if (p0 != p1 && (p0 != 0 || p1 != std::numeric_limits<llama_pos>::max())) {+ return false;+ }+ }+ }+for (uint32_t i = 0; i < cache.size; ++i) {if (cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {if (seq_id < 0) {@@ -1966,6 +2231,8 @@ static void llama_kv_cache_seq_rm(// If we freed up a slot, set head to it so searching can start there.if (new_head != cache.size && new_head < cache.head) cache.head = new_head;++ return true;}static void llama_kv_cache_seq_cp(@@ -1977,6 +2244,29 @@ static void llama_kv_cache_seq_cp(if (p0 < 0) p0 = 0;if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();+ if (cache.recurrent) {+ if ((uint32_t) seq_id_dst < cache.size && (uint32_t) seq_id_src < cache.size) {+ seq_id_src = cache.cells[seq_id_src].src;+ GGML_ASSERT((uint32_t) seq_id_src < cache.size);+ // intent to "copy from"+ // supports copy chains thanks to taking the source of the source+ cache.cells[seq_id_dst].src = seq_id_src;++ // preserve the "keep or clear" status of the copied sequence+ if (cache.cells[seq_id_src].has_seq_id(seq_id_src)) {+ cache.cells[seq_id_dst].seq_id.insert(seq_id_dst);+ } else {+ cache.cells[seq_id_dst].seq_id.erase(seq_id_dst);+ }++ cache.do_copy = true;++ cache.cells[seq_id_dst].pos = cache.cells[seq_id_src].pos;+ }+ return;+ }+ // otherwise, this is the KV cache of a Transformer-like model+cache.head = 0;for (uint32_t i = 0; i < cache.size; ++i) {@@ -2016,6 +2306,17 @@ static void llama_kv_cache_seq_add(if (p0 < 0) p0 = 0;if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();+ if (cache.recurrent) {+ // for Mamba-like models, only the pos needs to be shifted+ if (0 <= seq_id && seq_id < (int64_t) cache.size) {+ llama_kv_cell & cell = cache.cells[seq_id];+ if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) {+ cell.pos += delta;+ }+ }+ return;+ }+for (uint32_t i = 0; i < cache.size; ++i) {if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {cache.has_shift = true;@@ -2049,6 +2350,17 @@ static void llama_kv_cache_seq_div(if (p0 < 0) p0 = 0;if (p1 < 0) p1 = std::numeric_limits<llama_pos>::max();+ if (cache.recurrent) {+ // for Mamba-like models, only the pos needs to be changed+ if (0 <= seq_id && seq_id < (int64_t) cache.size) {+ llama_kv_cell & cell = cache.cells[seq_id];+ if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) {+ cell.pos /= d;+ }+ }+ return;+ }+for (uint32_t i = 0; i < cache.size; ++i) {if (cache.cells[i].has_seq_id(seq_id) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) {cache.has_shift = true;@@ -2692,7 +3004,11 @@ template<>bool llama_model_loader::get_key(const enum llm_kv kid, enum llama_pooling_type & result, const bool required) {uint32_t tmp;const bool found = get_key(kid, tmp, required);- result = (enum llama_pooling_type) tmp;+ if (found) {+ result = (enum llama_pooling_type) tmp;+ } else {+ result = LLAMA_POOLING_TYPE_UNSPECIFIED;+ }return found;}@@ -2770,6 +3086,7 @@ static const char * llama_model_type_name(e_model type) {case MODEL_20B: return "20B";case MODEL_30B: return "30B";case MODEL_34B: return "34B";+ case MODEL_35B: return "35B";case MODEL_40B: return "40B";case MODEL_65B: return "65B";case MODEL_70B: return "70B";@@ -2783,10 +3100,11 @@ static const char * llama_model_type_name(e_model type) {static const char * llama_model_vocab_type_name(enum llama_vocab_type type){switch (type) {- case LLAMA_VOCAB_TYPE_SPM: return "SPM";- case LLAMA_VOCAB_TYPE_BPE: return "BPE";- case LLAMA_VOCAB_TYPE_WPM: return "WPM";- default: return "unknown";+ case LLAMA_VOCAB_TYPE_NONE: return "no vocab";+ case LLAMA_VOCAB_TYPE_SPM: return "SPM";+ case LLAMA_VOCAB_TYPE_BPE: return "BPE";+ case LLAMA_VOCAB_TYPE_WPM: return "WPM";+ default: return "unknown";}}@@ -2818,14 +3136,14 @@ static void llm_load_hparams(ml.get_key(LLM_KV_GENERAL_NAME, model.name, false);// get hparams kv- ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);- ml.get_key (LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);- ml.get_key (LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);- ml.get_key (LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff);- ml.get_key (LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head);- ml.get_key (LLM_KV_BLOCK_COUNT, hparams.n_layer);- ml.get_key (LLM_KV_EXPERT_COUNT, hparams.n_expert, false);- ml.get_key (LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);+ ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);+ ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);+ ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);+ ml.get_key(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff);+ ml.get_key(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head);+ ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer);+ ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);+ ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);@@ -2865,7 +3183,7 @@ static void llm_load_hparams(// sanity check for n_rot (optional){- hparams.n_rot = hparams.n_embd / hparams.n_head;+ hparams.n_rot = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head;ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot, false);@@ -2878,10 +3196,10 @@ static void llm_load_hparams(// gpt-j n_rot = rotary_dim}- hparams.n_embd_head_k = hparams.n_embd / hparams.n_head;+ hparams.n_embd_head_k = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head;ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false);- hparams.n_embd_head_v = hparams.n_embd / hparams.n_head;+ hparams.n_embd_head_v = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head;ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false);// arch-specific KVs@@ -2969,7 +3287,7 @@ static void llm_load_hparams(ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type);- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type);+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);switch (hparams.n_layer) {case 3:@@ -3121,6 +3439,55 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_STARCODER2:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ switch (hparams.n_layer) {+ case 30: model.type = e_model::MODEL_3B; break;+ case 32: model.type = e_model::MODEL_7B; break;+ case 40: model.type = e_model::MODEL_15B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;+ case LLM_ARCH_MAMBA:+ {+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);+ ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);+ ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);+ ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);++ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);++ switch (hparams.n_layer) {+ case 24:+ switch (hparams.n_embd) {+ case 768: model.type = e_model::MODEL_SMALL; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ } break;+ case 48:+ switch (hparams.n_embd) {+ case 1024: model.type = e_model::MODEL_MEDIUM; break;+ case 1536: model.type = e_model::MODEL_LARGE; break;+ case 2048: model.type = e_model::MODEL_XL; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ } break;+ case 64:+ switch (hparams.n_embd) {+ case 2560: model.type = e_model::MODEL_3B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ } break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;+ case LLM_ARCH_COMMAND_R:+ {+ ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ switch (hparams.n_layer) {+ case 40: model.type = e_model::MODEL_35B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;default: (void)0;}@@ -3146,30 +3513,25 @@ static void llm_load_vocab(const auto kv = LLM_KV(model.arch);- const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());- if (token_idx == -1) {- throw std::runtime_error("cannot find tokenizer vocab in model file\n");- }-- const float * scores = nullptr;- const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());- if (score_idx != -1) {- scores = (const float * ) gguf_get_arr_data(ctx, score_idx);- }-- const int * toktypes = nullptr;- const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());- if (toktype_idx != -1) {- toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);- }-// determine vocab type{std::string tokenizer_name;ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_name);- if (tokenizer_name == "llama") {+ if (tokenizer_name == "no_vocab") {+ vocab.type = LLAMA_VOCAB_TYPE_NONE;++ // default special tokens+ vocab.special_bos_id = -1;+ vocab.special_eos_id = -1;+ vocab.special_unk_id = -1;+ vocab.special_sep_id = -1;+ vocab.special_pad_id = -1;+ vocab.linefeed_id = -1;++ return;+ } else if (tokenizer_name == "llama") {vocab.type = LLAMA_VOCAB_TYPE_SPM;// default special tokens@@ -3196,7 +3558,7 @@ static void llm_load_vocab(for (int i = 0; i < n_merges; i++) {const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);- GGML_ASSERT(codepoints_from_utf8(word).size() > 0);+ GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0);std::string first;std::string second;@@ -3235,13 +3597,30 @@ static void llm_load_vocab(}}+ const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());+ if (token_idx == -1) {+ throw std::runtime_error("cannot find tokenizer vocab in model file\n");+ }++ const float * scores = nullptr;+ const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());+ if (score_idx != -1) {+ scores = (const float * ) gguf_get_arr_data(ctx, score_idx);+ }++ const int * toktypes = nullptr;+ const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());+ if (toktype_idx != -1) {+ toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);+ }+const uint32_t n_vocab = gguf_get_arr_n(ctx, token_idx);vocab.id_to_token.resize(n_vocab);for (uint32_t i = 0; i < n_vocab; i++) {std::string word = gguf_get_arr_str(ctx, token_idx, i);- GGML_ASSERT(codepoints_from_utf8(word).size() > 0);+ GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0);vocab.token_to_id[word] = i;@@ -3430,9 +3809,11 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: f_norm_rms_eps = %.1e\n", __func__, hparams.f_norm_rms_eps);LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv);LLAMA_LOG_INFO("%s: f_max_alibi_bias = %.1e\n", __func__, hparams.f_max_alibi_bias);+ LLAMA_LOG_INFO("%s: f_logit_scale = %.1e\n", __func__, hparams.f_logit_scale);LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, hparams.n_ff);LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert);LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used);+ LLAMA_LOG_INFO("%s: causal attn = %d\n", __func__, hparams.causal_attn);LLAMA_LOG_INFO("%s: pooling type = %d\n", __func__, hparams.pooling_type);LLAMA_LOG_INFO("%s: rope type = %d\n", __func__, hparams.rope_type);LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type);@@ -3440,6 +3821,10 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train);LLAMA_LOG_INFO("%s: n_yarn_orig_ctx = %u\n", __func__, hparams.n_yarn_orig_ctx);LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown");+ LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);+ LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);+ LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);+ LLAMA_LOG_INFO("%s: ssm_dt_rank = %u\n", __func__, hparams.ssm_dt_rank);LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type));LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str());if (ml.n_elements >= 1e12) {@@ -3493,6 +3878,7 @@ static bool llm_load_tensors(// there is very little benefit to offloading the input layer, so always keep it on the CPUmodel.buft_input = llama_default_buffer_type_cpu(true);+ //model.buft_input = llama_default_buffer_type_offload(main_gpu);model.buft_layer.resize(n_layer);@@ -3626,7 +4012,13 @@ static bool llm_load_tensors({model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});if (model.arch != LLM_ARCH_MINICPM){- model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false);+ // if output is NULL, init from the input tok embed+ if (model.output == NULL) {+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ ml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);+ }}}@@ -3709,9 +4101,9 @@ static bool llm_load_tensors({model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});- if (gguf_find_tensor(ml.ctx_gguf, tn(LLM_TENSOR_OUTPUT, "weight").c_str()) >= 0) {- model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});- } else {++ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false);+ if (!model.output) {model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPUml.n_created--; // artificial tensorml.size_data += ggml_nbytes(model.output);@@ -3916,10 +4308,12 @@ static bool llm_load_tensors(model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, false);- // same as tok_embd, duplicated to allow offloading- model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});- ml.n_created--; // artificial tensor- ml.size_data += ggml_nbytes(model.output);+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false);+ if (!model.output) {+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU+ ml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);+ }}for (int i = 0; i < n_layer; ++i) {@@ -4291,37 +4685,178 @@ static bool llm_load_tensors(layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});}} break;- default:- throw std::runtime_error("unknown architecture");- }- }+ case LLM_ARCH_STARCODER2:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});- ml.done_getting_tensors();+ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});- ml.init_mapping(true, use_mlock ? &model.mlock_mmap : nullptr);+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false);+ // if output is NULL, init from the input tok embed+ if (model.output == NULL) {+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ ml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);+ }- // create the backend buffers- std::vector<std::pair<ggml_context *, ggml_backend_buffer_t>> ctx_bufs;+ }- for (auto & it : ctx_map) {- ggml_backend_buffer_type_t buft = it.first;- ggml_context * ctx = it.second;- ggml_backend_buffer_t buf = nullptr;+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);- // only the mmap region containing the tensors in the model is mapped to the backend buffer- // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers- // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size- if (ml.use_mmap && buft == llama_default_buffer_type_cpu(true)) {- size_t first, last;- ml.get_mapping_range(&first, &last, ctx);- buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first);- }+ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});++ // optional bias tensors+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});++ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});++ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});++ // optional bias tensors+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP , "bias", i), { n_ff});+ }+ } break;+ case LLM_ARCH_MAMBA:+ {+ const int64_t d_conv = hparams.ssm_d_conv;+ const int64_t d_inner = hparams.ssm_d_inner;+ const int64_t d_state = hparams.ssm_d_state;+ const int64_t dt_rank = hparams.ssm_dt_rank;+ // only an expansion factor of 2 is supported for now+ GGML_ASSERT(2 * n_embd == d_inner);++ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});++ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false);+ // if output is NULL, init from the input tok embed, duplicated to allow offloading+ if (model.output == NULL) {+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ ml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);+ }+ }++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ // norm+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});++ layer.ssm_in = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_IN, "weight", i), {n_embd, 2*d_inner});++ layer.ssm_conv1d = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, d_inner});+ layer.ssm_conv1d_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {d_inner});++ layer.ssm_x = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_X, "weight", i), {d_inner, dt_rank + 2*d_state});++ layer.ssm_dt = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_DT, "weight", i), {dt_rank, d_inner});+ layer.ssm_dt_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner});++ // no "weight" suffix for these+ layer.ssm_a = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_A, i), {d_state, d_inner});+ layer.ssm_d = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_SSM_D, i), {d_inner});++ // out_proj+ layer.ssm_out = ml.create_tensor(ctx_split, tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd});+ }+ } break;+ case LLM_ARCH_COMMAND_R:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ // init output from the input tok embed+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ ml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);+ }++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});++ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ }+ } break;+ default:+ throw std::runtime_error("unknown architecture");+ }+ }++ ml.done_getting_tensors();++ ml.init_mapping(true, use_mlock ? &model.mlock_mmap : nullptr);++ // create the backend buffers+ std::vector<std::pair<ggml_context *, ggml_backend_buffer_t>> ctx_bufs;++ for (auto & it : ctx_map) {+ ggml_backend_buffer_type_t buft = it.first;+ ggml_context * ctx = it.second;+ ggml_backend_buffer_t buf = nullptr;++ // only the mmap region containing the tensors in the model is mapped to the backend buffer+ // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers+ // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size+ if (ml.use_mmap && buft == llama_default_buffer_type_cpu(true)) {+ size_t first, last;+ ml.get_mapping_range(&first, &last, ctx);+ buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first);+#ifdef GGML_USE_CUBLAS+ if (n_layer >= n_gpu_layers) {+ ggml_backend_cuda_register_host_buffer(+ ggml_backend_buffer_get_base(buf),+ ggml_backend_buffer_get_size(buf));+ }+#endif+ }+#ifdef GGML_USE_METALelse if (ml.use_mmap && buft == ggml_backend_metal_buffer_type()) {const size_t max_size = ggml_get_max_tensor_size(ctx);size_t first, last;ml.get_mapping_range(&first, &last, ctx);buf = ggml_backend_metal_buffer_from_ptr((char *) ml.mapping->addr + first, last - first, max_size);}+#endifelse {buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);if (buf != nullptr && use_mlock && ggml_backend_buffer_is_host(buf)) {@@ -4409,7 +4944,8 @@ static int llama_model_load(const std::string & fname, llama_model & model, llamllm_load_print_meta(ml, model);- if (model.hparams.n_vocab != model.vocab.id_to_token.size()) {+ if (model.vocab.type != LLAMA_VOCAB_TYPE_NONE &&+ model.hparams.n_vocab != model.vocab.id_to_token.size()) {throw std::runtime_error("vocab size mismatch");}@@ -4434,6 +4970,16 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam}#endif+#ifdef GGML_USE_SYCL+ if (params.split_mode == LLAMA_SPLIT_MODE_NONE) {+ ggml_backend_sycl_set_single_device_mode(params.main_gpu);+ //SYCL use device index (0, 1, 2) directly, uer input device id, then convert to device index.+ params.main_gpu = ggml_backend_sycl_get_device_index(params.main_gpu);+ } else {+ ggml_backend_sycl_set_mul_device_mode();+ }+#endif+if (!llm_load_tensors(ml, model, params.n_gpu_layers, params.split_mode, params.main_gpu, params.tensor_split, params.use_mlock,params.progress_callback, params.progress_callback_user_data@@ -4473,29 +5019,32 @@ enum llm_norm_type {static struct ggml_tensor * llm_build_inp_embd(struct ggml_context * ctx,+ struct llama_context & lctx,const llama_hparams & hparams,const llama_batch & batch,struct ggml_tensor * tok_embd,- struct ggml_tensor * inp_tokens,- struct ggml_tensor * inp_embd,const llm_build_cb & cb) {const int64_t n_embd = hparams.n_embd;struct ggml_tensor * inpL;if (batch.token) {- struct ggml_tensor * inp_tokens_v = ggml_view_1d(ctx, inp_tokens, batch.n_tokens, 0);- cb(inp_tokens, "inp_tokens", -1);+ lctx.inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, batch.n_tokens);+ cb(lctx.inp_tokens, "inp_tokens", -1);+ ggml_set_input(lctx.inp_tokens);- inpL = ggml_get_rows(ctx, tok_embd, inp_tokens_v);+ inpL = ggml_get_rows(ctx, tok_embd, lctx.inp_tokens);} else {#ifdef GGML_USE_MPIGGML_ASSERT(false && "not implemented");#endif-- inpL = ggml_view_2d(ctx, inp_embd, n_embd, batch.n_tokens, inp_embd->nb[1], 0);+ lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens);+ inpL = lctx.inp_embd;+ ggml_set_input(lctx.inp_embd);}+ cb(inpL, "inp_embd", -1);+return inpL;}@@ -4514,6 +5063,8 @@ static void llm_build_kv_store(const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(kv.size == n_ctx);+// compute the transposed [n_tokens, n_embd] V matrixstruct ggml_tensor * v_cur_t = ggml_transpose(ctx, ggml_reshape_2d(ctx, v_cur, n_embd_v_gqa, n_tokens));//struct ggml_tensor * v_cur_t = ggml_transpose(ctx, v_cur); // TODO: reshape above is likely not needed@@ -4700,8 +5251,8 @@ static struct ggml_tensor * llm_build_kqv(ggml_mul_mat_set_prec(kq, GGML_PREC_F32);}-#if defined(GGML_USE_VULKAN) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_SYCL)-#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Vulkan, Kompute, and SYCL")+#if defined(GGML_USE_KOMPUTE)+#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Kompute")#pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024")#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488")if (hparams.f_max_alibi_bias > 0.0f) {@@ -4723,6 +5274,8 @@ static struct ggml_tensor * llm_build_kqv(cb(kq, "kq_soft_max_ext", il);}+ GGML_ASSERT(kv.size == n_ctx);+// split cached v into n_head headsstruct ggml_tensor * v =ggml_view_3d(ctx, kv.v_l[il],@@ -4785,6 +5338,7 @@ static struct ggml_tensor * llm_build_kv(llm_build_kv_store(ctx, hparams, kv, graph, k_cur, v_cur, n_ctx, n_tokens, kv_head, cb, il);struct ggml_tensor * cur;+cur = llm_build_kqv(ctx, model, hparams, kv, graph, wo, wo_b,q_cur, kq_mask, kq_pos, n_ctx, n_tokens, n_kv, kq_scale, cb, il);cb(cur, "kqv_out", il);@@ -4794,7 +5348,7 @@ static struct ggml_tensor * llm_build_kv(struct llm_build_context {const llama_model & model;- const llama_context & lctx;+ llama_context & lctx;const llama_hparams & hparams;const llama_cparams & cparams;const llama_batch & batch;@@ -4869,10 +5423,10 @@ struct llm_build_context {norm_eps (hparams.f_norm_eps),norm_rms_eps (hparams.f_norm_rms_eps),n_tokens (batch.n_tokens),- n_kv (worst_case ? n_ctx : kv_self.n),- kv_head (worst_case ? n_ctx - n_tokens : kv_self.head),+ n_kv (worst_case ? kv_self.size : kv_self.n),+ kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head),n_orig_ctx (cparams.n_yarn_orig_ctx),- pooling_type (cparams.do_pooling ? hparams.pooling_type : LLAMA_POOLING_TYPE_NONE),+ pooling_type (cparams.pooling_type),rope_type (hparams.rope_type),cb (cb),buf_compute_meta (lctx.buf_compute_meta) {@@ -4887,6 +5441,18 @@ struct llm_build_context {};ctx0 = ggml_init(params);++ lctx.inp_tokens = nullptr;+ lctx.inp_embd = nullptr;+ lctx.inp_pos = nullptr;+ lctx.inp_KQ_mask = nullptr;+ lctx.inp_KQ_pos = nullptr;+ lctx.inp_K_shift = nullptr;+ lctx.inp_mean = nullptr;+ lctx.inp_cls = nullptr;+ lctx.inp_s_copy = nullptr;+ lctx.inp_s_mask = nullptr;+ lctx.inp_s_seq = nullptr;}void free() {@@ -4899,6 +5465,12 @@ struct llm_build_context {struct ggml_cgraph * build_k_shift() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ GGML_ASSERT(kv_self.size == n_ctx);++ lctx.inp_K_shift = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_ctx);+ cb(lctx.inp_K_shift, "K_shift", -1);+ ggml_set_input(lctx.inp_K_shift);+for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * tmp =// we rotate only the first n_rot dimensions@@ -4917,6 +5489,29 @@ struct llm_build_context {return gf;}+ struct ggml_cgraph * build_s_copy() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ GGML_ASSERT(kv_self.recurrent);++ struct ggml_tensor * state_copy = build_inp_s_copy();++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * conv_states = ggml_reshape_2d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s(), kv_self.size);+ struct ggml_tensor * ssm_states = ggml_reshape_2d(ctx0, kv_self.v_l[il], hparams.n_embd_v_s(), kv_self.size);++ conv_states = ggml_get_rows(ctx0, conv_states, state_copy);+ ssm_states = ggml_get_rows(ctx0, ssm_states, state_copy);++ // TODO: name the intermediate tensors with cb()++ ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_states, kv_self.k_l[il]));+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, ssm_states, kv_self.v_l[il]));+ }++ return gf;+ }+struct ggml_cgraph * build_defrag(const std::vector<uint32_t> & ids) {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);@@ -4966,6 +5561,66 @@ struct llm_build_context {return gf;}+ struct ggml_tensor * build_inp_pos() {+ lctx.inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ cb(lctx.inp_pos, "inp_pos", -1);+ ggml_set_input(lctx.inp_pos);+ return lctx.inp_pos;+ }++ struct ggml_tensor * build_inp_KQ_mask(bool causal = true) {+ if (causal) {+ lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_tokens);+ } else {+ lctx.inp_KQ_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens);+ }+ cb(lctx.inp_KQ_mask, "KQ_mask", -1);+ ggml_set_input(lctx.inp_KQ_mask);+ return lctx.inp_KQ_mask;+ }++ struct ggml_tensor * build_inp_KQ_pos() {+ lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_kv);+ cb(lctx.inp_KQ_pos, "KQ_pos", -1);+ ggml_set_input(lctx.inp_KQ_pos);+ return lctx.inp_KQ_pos;+ }++ struct ggml_tensor * build_inp_mean() {+ lctx.inp_mean = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens);+ cb(lctx.inp_mean, "inp_mean", -1);+ ggml_set_input(lctx.inp_mean);+ return lctx.inp_mean;+ }++ struct ggml_tensor * build_inp_cls() {+ lctx.inp_cls = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ cb(lctx.inp_cls, "inp_cls", -1);+ ggml_set_input(lctx.inp_cls);+ return lctx.inp_cls;+ }++ struct ggml_tensor * build_inp_s_copy() {+ lctx.inp_s_copy = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, kv_self.size);+ cb(lctx.inp_s_copy, "inp_s_copy", -1);+ ggml_set_input(lctx.inp_s_copy);+ return lctx.inp_s_copy;+ }++ struct ggml_tensor * build_inp_s_mask() {+ lctx.inp_s_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_kv);+ cb(lctx.inp_s_mask, "inp_s_mask", -1);+ ggml_set_input(lctx.inp_s_mask);+ return lctx.inp_s_mask;+ }++ struct ggml_tensor * build_inp_s_seq() {+ lctx.inp_s_seq = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_tokens);+ cb(lctx.inp_s_seq, "inp_s_seq", -1);+ ggml_set_input(lctx.inp_s_seq);+ return lctx.inp_s_seq;+ }+struct ggml_cgraph * build_llama() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);@@ -4976,16 +5631,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -5037,7 +5689,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -5123,6 +5774,12 @@ struct llm_build_context {}cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "ffn_out", il);++ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);+ if (layer_dir != nullptr) {+ cur = ggml_add(ctx0, cur, layer_dir);+ }cb(cur, "l_out", il);// input for next layer@@ -5155,20 +5812,16 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = model.type == MODEL_7B ? build_inp_pos() : nullptr;// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();// positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);- cb(KQ_pos, "KQ_pos", -1);+ struct ggml_tensor * KQ_pos = build_inp_KQ_pos();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -5212,11 +5865,9 @@ struct llm_build_context {cb(Qcur, "Qcur", il);cb(Kcur, "Kcur", il);-cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -5272,16 +5923,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * attn_norm;@@ -5335,7 +5983,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = cur;@@ -5386,21 +6033,17 @@ struct llm_build_context {GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);struct ggml_tensor * cur;- struct ggml_tensor * pos;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);+ struct ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);cb(pos, "pos_embd", -1);inpL = ggml_add(ctx0, inpL, pos);@@ -5434,7 +6077,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}// add the input@@ -5486,16 +6128,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * residual = inpL;@@ -5635,7 +6274,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Q, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur);@@ -5689,16 +6327,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();// positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);- cb(KQ_pos, "KQ_pos", -1);+ struct ggml_tensor * KQ_pos = build_inp_KQ_pos();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -5728,7 +6363,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -5778,19 +6412,18 @@ struct llm_build_context {const int64_t n_embd_head = hparams.n_embd_head_v;const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);struct ggml_tensor * cur;struct ggml_tensor * inpL;- // get input vectors with right size- const size_t stride1 = n_tokens * ggml_type_size(lctx.inp_tokens->type);- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- struct ggml_tensor * inp_mean = ggml_view_2d(ctx0, lctx.inp_mean, n_tokens, n_tokens, stride1, 0);- struct ggml_tensor * inp_cls = ggml_view_1d(ctx0, lctx.inp_cls, n_tokens, 0);+ struct ggml_tensor * inp_pos = build_inp_pos();+ struct ggml_tensor * inp_mean = build_inp_mean();+ struct ggml_tensor * inp_cls = build_inp_cls();// construct input embeddings (token, type, position)- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// token types are hardcoded to zero ("Sentence A")struct ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0);@@ -5805,39 +6438,37 @@ struct llm_build_context {cb(inpL, "inp_norm", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1); // [n_kv, n_tokens]+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask(false);// iterate layersfor (int il = 0; il < n_layer; ++il) {struct ggml_tensor * cur = inpL;+ struct ggml_tensor * Qcur;+ struct ggml_tensor * Kcur;+ struct ggml_tensor * Vcur;+// self-attentionif (model.arch == LLM_ARCH_BERT) {- struct ggml_tensor * Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq);+ Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq);cb(Qcur, "Qcur", il);- struct ggml_tensor * Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk);+ Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk);cb(Kcur, "Kcur", il);- struct ggml_tensor * Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv);+ Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv);cb(Vcur, "Vcur", il);- // seems like we just need to do this for Q?- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);-- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,- model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);} else {// compute Q and K and RoPE themcur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur);cb(cur, "wqkv", il);- struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));- struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));- struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));+ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));cb(Qcur, "Qcur", il);cb(Kcur, "Kcur", il);@@ -5856,12 +6487,40 @@ struct llm_build_context {ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);+ }- cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,- model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));++ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);+ cb(kq, "kq", il);++ kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, nullptr, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias);+ cb(kq, "kq_soft_max_ext", il);++ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));+ cb(v, "v", il);++ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq);+ cb(kqv, "kqv", il);++ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);+ cb(kqv_merged, "kqv_merged", il);++ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);+ cb(cur, "kqv_merged_cont", il);++ ggml_build_forward_expand(gf, cur);++ cur = ggml_mul_mat(ctx0, model.layers[il].wo, cur);+ if (model.layers[il].bo) {+ cb(cur, "kqv_wo", il);+ }++ if (model.layers[il].bo) {+ cur = ggml_add(ctx0, cur, model.layers[il].bo);}+ cb(cur, "kqv_out", il);// re-add the layer inputcur = ggml_add(ctx0, cur, inpL);@@ -5902,16 +6561,29 @@ struct llm_build_context {// final outputcur = inpL;+ cb(cur, "result_embd", -1);// pooling layer- if (pooling_type == LLAMA_POOLING_TYPE_MEAN) {- cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean);- } else if (pooling_type == LLAMA_POOLING_TYPE_CLS) {- cur = ggml_get_rows(ctx0, cur, inp_cls);- } else {- GGML_ASSERT(pooling_type == LLAMA_POOLING_TYPE_NONE && "Invalid pooling type");+ switch (pooling_type) {+ case LLAMA_POOLING_TYPE_NONE:+ {+ // nop+ } break;+ case LLAMA_POOLING_TYPE_MEAN:+ {+ cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean);+ cb(cur, "result_embd_pooled", -1);+ } break;+ case LLAMA_POOLING_TYPE_CLS:+ {+ cur = ggml_get_rows(ctx0, cur, inp_cls);+ cb(cur, "result_embd_pooled", -1);+ } break;+ case LLAMA_POOLING_TYPE_UNSPECIFIED:+ {+ GGML_ASSERT(false && "Invalid pooling type");+ } break;}- cb(cur, "result_embd", -1);ggml_build_forward_expand(gf, cur);@@ -5928,16 +6600,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();// positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);- cb(KQ_pos, "KQ_pos", -1);+ struct ggml_tensor * KQ_pos = build_inp_KQ_pos();inpL = llm_build_norm(ctx0, inpL, hparams,model.tok_norm,@@ -5973,7 +6642,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}// Add the input@@ -6025,16 +6693,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();// positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = ggml_view_1d(ctx0, lctx.inp_KQ_pos, n_kv, 0);- cb(KQ_pos, "KQ_pos", -1);+ struct ggml_tensor * KQ_pos = build_inp_KQ_pos();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * attn_norm;@@ -6075,7 +6740,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}// Add the input@@ -6130,16 +6794,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -6192,7 +6853,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -6248,16 +6908,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -6302,7 +6959,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -6357,16 +7013,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -6418,7 +7071,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -6473,16 +7125,13 @@ struct llm_build_context {struct ggml_tensor * ffn_output;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {attn_norm_output = llm_build_norm(ctx0, inpL, hparams,@@ -6540,7 +7189,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);- cb(cur, "kqv_out", il);}// FF@@ -6590,16 +7238,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {@@ -6638,7 +7283,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * sa_out = cur;@@ -6692,16 +7336,13 @@ struct llm_build_context {struct ggml_tensor * pos;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);cb(pos, "pos_embd", -1);@@ -6737,7 +7378,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}// add the input@@ -6790,16 +7430,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {cur = llm_build_norm(ctx0, inpL, hparams,@@ -6841,7 +7478,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}// add the input@@ -6893,16 +7529,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -6954,7 +7587,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -7007,16 +7639,13 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -7068,7 +7697,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);@@ -7130,20 +7758,17 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);// scale the input embeddingsinpL = ggml_scale(ctx0, inpL, scale_embd);cb(inpL, "inp_scaled", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -7195,7 +7820,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- cb(cur, "kqv_out", il);}// scale_res - scale the hidden states for residual connection@@ -7262,22 +7886,18 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);- cb(inpL, "inp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));cb(inpL, "inp_scaled", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);- cb(inp_pos, "inp_pos", -1);+ struct ggml_tensor * inp_pos = build_inp_pos();// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);- cb(KQ_mask, "KQ_mask", -1);+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();for (int il = 0; il < n_layer; ++il) {-// normcur = llm_build_norm(ctx0, inpL, hparams,model.layers[il].attn_norm, NULL,@@ -7314,7 +7934,6 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);- cb(cur, "kqv_out", il);}struct ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);@@ -7358,36 +7977,416 @@ struct llm_build_context {return gf;}-};--static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {- llama_batch dummy;- dummy.n_tokens = 0;-- llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { };- struct llm_build_context llm(lctx, dummy, cb, false);+ struct ggml_cgraph * build_starcoder2() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);- llm.init();+ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);- struct ggml_cgraph * result = llm.build_defrag(ids);+ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;- llm.free();+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);- return result;-}+ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = build_inp_pos();-static struct ggml_cgraph * llama_build_graph_k_shift(llama_context & lctx) {- llama_batch dummy;- dummy.n_tokens = 0;+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { };+ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;- struct llm_build_context llm(lctx, dummy, cb, false);+ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, model.layers[il].attn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "attn_norm", il);- llm.init();+ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ if (model.layers[il].bq) {+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);+ }- struct ggml_cgraph * result = llm.build_k_shift();+ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ if (model.layers[il].bk) {+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);+ }++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ if (model.layers[il].bv) {+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);+ }++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ }++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network++ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, model.layers[il].ffn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, model.layers[il].ffn_up_b,+ NULL, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);+ cb(cur, "ffn_out", il);+ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, model.output_norm_b,+ LLM_NORM, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }++ struct ggml_cgraph * build_mamba() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t d_model = n_embd;+ const int64_t d_conv = hparams.ssm_d_conv;+ const int64_t d_inner = hparams.ssm_d_inner;+ GGML_ASSERT(2 * d_model == d_inner);+ const int64_t d_state = hparams.ssm_d_state;+ const int64_t dt_rank = hparams.ssm_dt_rank;++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ // {n_embd, n_tokens}+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);++ struct ggml_tensor * state_mask = build_inp_s_mask();+ struct ggml_tensor * state_seq = build_inp_s_seq();++ for (int il = 0; il < n_layer; ++il) {+ // (ab)using the KV cache to store the states+ struct ggml_tensor * conv_states = ggml_reshape_2d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s(), kv_self.size);+ struct ggml_tensor * ssm_states = ggml_reshape_2d(ctx0, kv_self.v_l[il], hparams.n_embd_v_s(), kv_self.size);++ // clear states of sequences which are starting at the beginning of this batch+ {+ conv_states = ggml_mul(ctx0,+ ggml_view_2d(ctx0, conv_states, conv_states->ne[0], n_kv, conv_states->nb[1], kv_head*conv_states->nb[1]),+ state_mask);+ ssm_states = ggml_mul(ctx0,+ ggml_view_2d(ctx0, ssm_states, ssm_states->ne[0], n_kv, ssm_states->nb[1], kv_head*ssm_states->nb[1]),+ state_mask);+ }++ conv_states = ggml_reshape_3d(ctx0, conv_states, d_conv - 1, d_inner, n_kv);+ ssm_states = ggml_reshape_3d(ctx0, ssm_states, d_state, d_inner, n_kv);++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ // {n_embd, 2*d_inner} * {n_embd, n_tokens} => {2*d_inner, n_tokens}+ struct ggml_tensor * xz = ggml_mul_mat(ctx0, model.layers[il].ssm_in, cur);+ // split the above in two+ // => {d_inner, n_tokens}+ struct ggml_tensor * x = ggml_view_2d(ctx0, xz, d_inner, xz->ne[1], xz->nb[1], 0);+ struct ggml_tensor * z = ggml_view_2d(ctx0, xz, d_inner, xz->ne[1], xz->nb[1], ggml_element_size(xz)*d_inner);++ // conv+ {+ // Custom operator which is needed only to ease simultaneous sequence processing.+ // For a single sequence, the equivalent is to concatenate the columns of conv_states and x,+ // then make a self-overlapping view of that over d_conv columns at each stride in the 3rd dimension,+ // then element-wise multiply that with the conv1d weigth,+ // then sum the elements of each row,+ // (the last two steps are a dot product over rows (also doable with mul_mat))+ // then permute away the ne[0] dimension,+ // and then you're left with the resulting x tensor.+ // The new conv_states is the last (d_conv - 1) columns+ // of the last 3rd dimensional "layer" of the self-overlapping view.+ // For simultaneous sequences, it's more complicated.+ struct ggml_tensor * x_conv = ggml_ssm_conv(ctx0, conv_states, x, model.layers[il].ssm_conv1d, state_seq);++ // store last (d_conv - 1) columns of the conv_state part of x_conv back into the KV cache+ ggml_build_forward_expand(gf,+ ggml_cpy(ctx0,+ ggml_view_2d(ctx0, x_conv, d_conv - 1, d_inner*n_kv, d_conv*ggml_element_size(x_conv), (1+d_inner*n_tokens)*ggml_element_size(x_conv)),+ ggml_view_1d(ctx0, kv_self.k_l[il], (d_conv - 1)*(d_inner)*(n_kv), kv_head*(d_conv - 1)*(d_inner)*ggml_element_size(x_conv))));++ // extract x from x_conv+ x = ggml_view_2d(ctx0, x_conv, d_inner, n_tokens, d_inner*ggml_element_size(x_conv), 0);++ // bias+ x = ggml_add(ctx0, x, model.layers[il].ssm_conv1d_b);++ x = ggml_silu(ctx0, x);+ }++ // ssm+ {+ // {d_inner, dt_rank + 2*d_state} * {d_inner, n_tokens} => {dt_rank + 2*d_state, n_tokens}+ struct ggml_tensor * x_db = ggml_mul_mat(ctx0, model.layers[il].ssm_x, x);+ // split+ struct ggml_tensor * dt = ggml_view_2d(ctx0, x_db, dt_rank, n_tokens, x_db->nb[1], 0);+ struct ggml_tensor * B = ggml_view_2d(ctx0, x_db, d_state, n_tokens, x_db->nb[1], ggml_element_size(x_db)*dt_rank);+ struct ggml_tensor * C = ggml_view_2d(ctx0, x_db, d_state, n_tokens, x_db->nb[1], ggml_element_size(x_db)*(dt_rank+d_state));++ // {dt_rank, d_inner} * {dt_rank, n_tokens} => {d_inner, n_tokens}+ dt = ggml_mul_mat(ctx0, model.layers[il].ssm_dt, dt);+ dt = ggml_add(ctx0, dt, model.layers[il].ssm_dt_b);++ // Custom operator to optimize the parallel associative scan+ // as described in the Annex D of the Mamba paper.+ // => {d_inner, n_tokens} and {d_state, d_inner, n_kv} combined,+ // because only a single tensor can be returned.+ struct ggml_tensor * y_ssm_states = ggml_ssm_scan(ctx0, ssm_states, x, dt, model.layers[il].ssm_a, B, C, state_seq);++ // store last states (the second part of y_ssm_states)+ ggml_build_forward_expand(gf,+ ggml_cpy(ctx0,+ ggml_view_1d(ctx0, y_ssm_states, d_state*d_inner*n_kv, d_inner*n_tokens*ggml_element_size(y_ssm_states)),+ ggml_view_1d(ctx0, kv_self.v_l[il], d_state*d_inner*n_kv, kv_head*d_state*d_inner*ggml_element_size(ssm_states))));++ struct ggml_tensor * y = ggml_view_2d(ctx0, y_ssm_states, d_inner, n_tokens, d_inner*ggml_element_size(y_ssm_states), 0);++ // {d_inner, n_tokens} * {d_inner} => {d_inner, n_tokens}+ y = ggml_add(ctx0, y, ggml_mul(ctx0, x, model.layers[il].ssm_d));+ y = ggml_mul(ctx0, y, ggml_silu(ctx0, z));++ // {d_inner, n_embd} * {d_inner, n_tokens} => {n_embd, n_tokens}+ cur = ggml_mul_mat(ctx0, model.layers[il].ssm_out, y);+ }++ // residual+ cur = ggml_add(ctx0, cur, inpL);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ // final rmsnorm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }++ struct ggml_cgraph * build_command_r() {++ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ const float f_logit_scale = hparams.f_logit_scale;++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = build_inp_pos();++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();++ for (int il = 0; il < n_layer; ++il) {++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM, cb, il);+ cb(cur, "attn_norm", il);+ struct ggml_tensor * ffn_inp = cur;++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ if (model.layers[il].bq) {+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);+ }++ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ if (model.layers[il].bk) {+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);+ }++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ if (model.layers[il].bv) {+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);+ }++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ }++ struct ggml_tensor * attn_out = cur;++ // feed-forward network+ {+ cur = llm_build_ffn(ctx0, ffn_inp,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);+ }++ // add together residual + FFN + self-attention+ cur = ggml_add(ctx0, cur, inpL);+ cur = ggml_add(ctx0, cur, attn_out);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);++ if (f_logit_scale) {+ cur = ggml_scale(ctx0, cur, f_logit_scale);+ }++ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;++ }+};++static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {+ llama_batch dummy;+ dummy.n_tokens = 0;++ llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { };++ struct llm_build_context llm(lctx, dummy, cb, false);++ llm.init();++ struct ggml_cgraph * result = llm.build_defrag(ids);++ llm.free();++ return result;+}++static struct ggml_cgraph * llama_build_graph_k_shift(llama_context & lctx) {+ llama_batch dummy;+ dummy.n_tokens = 0;++ llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { };++ struct llm_build_context llm(lctx, dummy, cb, false);++ llm.init();++ struct ggml_cgraph * result = llm.build_k_shift();++ llm.free();++ return result;+}++static struct ggml_cgraph * llama_build_graph_s_copy(llama_context & lctx) {+ llama_batch dummy;+ dummy.n_tokens = 0;++ llm_build_cb cb = [&](struct ggml_tensor * , const char * , int ) { };++ struct llm_build_context llm(lctx, dummy, cb, false);++ llm.init();++ struct ggml_cgraph * result = llm.build_s_copy();llm.free();@@ -7411,7 +8410,21 @@ static struct ggml_cgraph * llama_build_graph(if (!lctx.cparams.offload_kqv) {if (strcmp(name, "kqv_merged_cont") == 0) {// all nodes between the KV store and the attention output are run on the CPU- ggml_backend_sched_set_node_backend(lctx.sched, cur, lctx.backend_cpu);+ ggml_backend_sched_set_tensor_backend(lctx.sched, cur, lctx.backend_cpu);+ }+ }++ // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends+ // FIXME: fix in ggml_backend_sched+ const bool full_offload = lctx.model.n_gpu_layers > (int)lctx.model.hparams.n_layer;+ if (batch.n_tokens < 32 || full_offload) {+ if (il != -1 && strcmp(name, "norm") == 0) {+ for (auto * backend : lctx.backends) {+ if (ggml_backend_buft_supports_backend(lctx.model.buft_layer[il].buft, backend)) {+ ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend);+ break;+ }+ }}}};@@ -7504,6 +8517,18 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_gemma();} break;+ case LLM_ARCH_STARCODER2:+ {+ result = llm.build_starcoder2();+ } break;+ case LLM_ARCH_MAMBA:+ {+ result = llm.build_mamba();+ } break;+ case LLM_ARCH_COMMAND_R:+ {+ result = llm.build_command_r();+ } break;default:GGML_ASSERT(false);}@@ -7514,19 +8539,29 @@ static struct ggml_cgraph * llama_build_graph(}static void llama_set_k_shift(llama_context & lctx) {- const auto & cparams = lctx.cparams;-- const int64_t n_ctx = cparams.n_ctx;+ const int64_t kv_size = lctx.kv_self.size;assert(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer));int32_t * data = (int32_t *) lctx.inp_K_shift->data;- for (int i = 0; i < n_ctx; ++i) {+ for (int i = 0; i < kv_size; ++i) {data[i] = lctx.kv_self.cells[i].delta;}}+static void llama_set_s_copy(llama_context & lctx) {+ const int64_t kv_size = lctx.kv_self.size;++ assert(ggml_backend_buffer_is_host(lctx.inp_s_copy->buffer));++ int32_t * data = (int32_t *) lctx.inp_s_copy->data;++ for (int i = 0; i < kv_size; ++i) {+ data[i] = lctx.kv_self.cells[i].src;+ }+}+static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {//// set input data@@ -7549,34 +8584,74 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd));}- if (batch.pos) {+ if (batch.pos && lctx.inp_pos) {const int64_t n_tokens = batch.n_tokens;ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos));}- {- const int64_t n_kv = kv_self.n;- const int64_t n_tokens = batch.n_tokens;+ GGML_ASSERT(+ (hparams.causal_attn || !cparams.causal_attn) &&+ "non-causal attention with generative models is not supported"+ );- assert(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));+ if (lctx.inp_KQ_mask) {+ // NOTE: hparams.causal_attn indicates the model is capable of generation and uses the kv cache.+ if (cparams.causal_attn) {+ const int64_t n_kv = kv_self.n;+ const int64_t n_tokens = batch.n_tokens;- float * data = (float *) lctx.inp_KQ_mask->data;+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));- for (int h = 0; h < 1; ++h) {- for (int j = 0; j < n_tokens; ++j) {- const llama_pos pos = batch.pos[j];- const llama_seq_id seq_id = batch.seq_id[j][0];+ float * data = (float *) lctx.inp_KQ_mask->data;- for (int i = 0; i < n_kv; ++i) {- float f;- if (!lctx.kv_self.cells[i].has_seq_id(seq_id) ||- (hparams.causal_attn && lctx.kv_self.cells[i].pos > pos)) {- f = -INFINITY;- } else {- f = 0;+ // For causal attention, use only the previous KV cells+ // of the correct sequence for each token of the batch.+ // It's assumed that if a token in the batch has multiple sequences, they are equivalent.+ for (int h = 0; h < 1; ++h) {+ for (int j = 0; j < n_tokens; ++j) {+ const llama_pos pos = batch.pos[j];+ const llama_seq_id seq_id = batch.seq_id[j][0];++ for (int i = 0; i < n_kv; ++i) {+ float f;+ if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) {+ f = -INFINITY;+ } else {+ f = 0.0f;+ }+ data[h*(n_kv*n_tokens) + j*n_kv + i] = f;+ }+ }+ }+ } else {+ // when using kv cache, the mask needs to match the kv cache size+ const int64_t n_tokens = batch.n_tokens;+ const int64_t n_stride = hparams.causal_attn ? kv_self.n : n_tokens;++ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));++ float * data = (float *) lctx.inp_KQ_mask->data;++ for (int h = 0; h < 1; ++h) {+ for (int j = 0; j < n_tokens; ++j) {+ const llama_seq_id seq_id = batch.seq_id[j][0];++ for (int i = 0; i < n_tokens; ++i) {+ float f = -INFINITY;+ for (int s = 0; s < batch.n_seq_id[i]; ++s) {+ if (batch.seq_id[i][s] == seq_id) {+ f = 0.0f;+ break;+ }+ }++ data[h*(n_tokens*n_tokens) + j*n_stride + i] = f;+ }++ for (int i = n_tokens; i < n_stride; ++i) {+ data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY;}- data[h*(n_kv*n_tokens) + j*n_kv + i] = f;}}}@@ -7585,7 +8660,8 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {if (hparams.need_kq_pos) {const int64_t n_kv = kv_self.n;- assert(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer));+ GGML_ASSERT(lctx.inp_KQ_pos);+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer));float * data = (float *) lctx.inp_KQ_pos->data;@@ -7594,17 +8670,21 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {}}- if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) {+ if (cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) {const int64_t n_tokens = batch.n_tokens;+ GGML_ASSERT(lctx.inp_mean);GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_mean->buffer));- float * data = (float *) lctx.inp_mean->data;+ float * data = (float *) lctx.inp_mean->data;memset(lctx.inp_mean->data, 0, n_tokens * n_tokens * ggml_element_size(lctx.inp_mean));std::vector<uint64_t> sum(n_tokens, 0);for (int i = 0; i < n_tokens; ++i) {const llama_seq_id seq_id = batch.seq_id[i][0];++ GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == MEAN");+sum[seq_id] += 1;}@@ -7622,20 +8702,73 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {}}- if (cparams.do_pooling && hparams.pooling_type == LLAMA_POOLING_TYPE_CLS) {+ if (cparams.pooling_type == LLAMA_POOLING_TYPE_CLS) {const int64_t n_tokens = batch.n_tokens;+ GGML_ASSERT(lctx.inp_cls);GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer));+uint32_t * data = (uint32_t *) lctx.inp_cls->data;+ memset(lctx.inp_cls->data, 0, n_tokens * ggml_element_size(lctx.inp_cls));for (int i = 0; i < n_tokens; ++i) {const llama_seq_id seq_id = batch.seq_id[i][0];- const llama_pos pos = batch.pos[i];+ const llama_pos pos = batch.pos[i];++ GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == CLS");+if (pos == 0) {data[seq_id] = i;}}}++ if (kv_self.recurrent) {+ const int64_t n_kv = kv_self.n;++ if (lctx.inp_s_mask) {+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_mask->buffer));+ float * data = (float *) lctx.inp_s_mask->data;++ // states which are not affected by the current batch are left untouched+ for (int i = 0; i < n_kv; ++i) {+ llama_seq_id seq_id = i + lctx.kv_self.head;+ llama_kv_cell & kv_cell = lctx.kv_self.cells[seq_id];+ bool has_self_seq = kv_cell.has_seq_id(seq_id);++ data[i] = (float) has_self_seq;++ // ensure current sequences will be kept+ if (!has_self_seq && kv_cell.pos >= 0) {+ kv_cell.seq_id.insert(seq_id);+ }+ }+ }+ // For Mamba (and other recurrent architectures),+ // update the correct state(s)/sequence(s) for each token of the batch.+ // Like with the KQ_mask, if a token in the batch has multiple sequences,+ // they are assumed to be equivalent (not here, but in ggml_ssm_scan and ggml_ssm_conv).+ if (lctx.inp_s_seq) {+ const int64_t n_tokens = batch.n_tokens;++ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_s_seq->buffer));+ int32_t * data = (int32_t *) lctx.inp_s_seq->data;++ for (int j = 0; j < n_tokens; ++j) {+ const int32_t n_seq = batch.n_seq_id[j];+ GGML_ASSERT(0 < n_seq); // a token should be part of at least 1 sequence++ for (int i = 0; i < n_kv; ++i) {+ if (i < n_seq) {+ // for this type of model, the head is the minimum seq_id of the batch+ data[j*n_kv + i] = batch.seq_id[j][i] - kv_self.head;+ } else {+ data[j*n_kv + i] = -1;+ }+ }+ }+ }+ }}static void llama_graph_compute(@@ -7653,9 +8786,10 @@ static void llama_graph_compute(if (lctx.backend_cpu != nullptr) {ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads);+ ggml_backend_cpu_set_abort_callback(lctx.backend_cpu, lctx.abort_callback, lctx.abort_callback_data);}- ggml_backend_sched_graph_compute(lctx.sched, gf);+ ggml_backend_sched_graph_compute_async(lctx.sched, gf);// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));@@ -7675,10 +8809,11 @@ static void llama_graph_compute(//static int llama_decode_internal(llama_context & lctx,- llama_batch batch) {- const uint32_t n_tokens = batch.n_tokens;+ llama_batch batch_all) { // TODO: rename back to batch- if (n_tokens == 0) {+ const uint32_t n_tokens_all = batch_all.n_tokens;++ if (n_tokens_all == 0) {LLAMA_LOG_ERROR("%s: n_tokens == 0", __func__);return -1;}@@ -7687,14 +8822,16 @@ static int llama_decode_internal(const auto & hparams = model.hparams;const auto & cparams = lctx.cparams;- const auto n_batch = cparams.n_batch;+ GGML_ASSERT((!batch_all.token && batch_all.embd) || (batch_all.token && !batch_all.embd)); // NOLINT- GGML_ASSERT(n_tokens <= n_batch);+ GGML_ASSERT(n_tokens_all <= cparams.n_batch);- int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;- GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT+ GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");- const int64_t t_start_us = ggml_time_us();+ if (lctx.t_compute_start_us == 0) {+ lctx.t_compute_start_us = ggml_time_us();+ }+ lctx.n_queued_tokens += n_tokens_all;#ifdef GGML_USE_MPI// TODO: needs fix after #3228@@ -7702,213 +8839,274 @@ static int llama_decode_internal(//ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads);#endif- GGML_ASSERT(n_threads > 0);-auto & kv_self = lctx.kv_self;const int64_t n_embd = hparams.n_embd;const int64_t n_vocab = hparams.n_vocab;- // helpers for smoother batch API transition- // after deprecating the llama_eval calls, these will be removed- std::vector<llama_pos> pos;+ auto * logits_out = lctx.logits;++#ifndef NDEBUG+ auto & logits_valid = lctx.logits_valid;+ logits_valid.clear();+ logits_valid.resize(n_tokens_all);++ memset(logits_out, 0, lctx.logits_size*sizeof(float));+#endif++ const auto n_ubatch = cparams.n_ubatch;++ std::vector<llama_pos> pos;std::vector<int32_t> n_seq_id;std::vector<llama_seq_id *> seq_id_arr;std::vector<std::vector<llama_seq_id>> seq_id;- if (batch.pos == nullptr) {- pos.resize(n_tokens);- for (uint32_t i = 0; i < n_tokens; i++) {- pos[i] = batch.all_pos_0 + i*batch.all_pos_1;- }+ for (uint32_t cur_token = 0; cur_token < n_tokens_all; cur_token += n_ubatch) {+ const uint32_t n_tokens = std::min(n_ubatch, n_tokens_all - cur_token);+ llama_batch u_batch = {+ /* .n_tokens = */ (int32_t) n_tokens,+ /* .token = */ batch_all.token ? batch_all.token + cur_token : nullptr,+ /* .embd = */ batch_all.embd ? batch_all.embd + cur_token*n_embd : nullptr,+ /* .pos = */ batch_all.pos ? batch_all.pos + cur_token : nullptr,+ /* .n_seq_id = */ batch_all.n_seq_id ? batch_all.n_seq_id + cur_token : nullptr,+ /* .seq_id = */ batch_all.seq_id ? batch_all.seq_id + cur_token : nullptr,+ /* .logits = */ batch_all.logits ? batch_all.logits + cur_token : nullptr,+ /* .all_pos_0 = */ batch_all.all_pos_0 + (llama_pos) cur_token*batch_all.all_pos_1,+ /* .all_pos_1 = */ batch_all.all_pos_1,+ /* .all_seq_id = */ batch_all.all_seq_id,+ };- batch.pos = pos.data();- }+ int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;+ GGML_ASSERT(n_threads > 0);- if (batch.seq_id == nullptr) {- n_seq_id.resize(n_tokens);- seq_id.resize(n_tokens);- seq_id_arr.resize(n_tokens);- for (uint32_t i = 0; i < n_tokens; i++) {- n_seq_id[i] = 1;- seq_id[i].resize(1);- seq_id[i][0] = batch.all_seq_id;- seq_id_arr[i] = seq_id[i].data();+ // helpers for smoother batch API transition+ // after deprecating the llama_eval calls, these will be removed+ if (u_batch.pos == nullptr) {+ pos.resize(n_tokens);+ for (uint32_t i = 0; i < n_tokens; i++) {+ pos[i] = u_batch.all_pos_0 + i*u_batch.all_pos_1;+ }++ u_batch.pos = pos.data();}- batch.n_seq_id = n_seq_id.data();- batch.seq_id = seq_id_arr.data();- }+ if (u_batch.seq_id == nullptr) {+ n_seq_id.resize(n_tokens);+ seq_id.resize(n_tokens);+ seq_id_arr.resize(n_tokens);+ for (uint32_t i = 0; i < n_tokens; i++) {+ n_seq_id[i] = 1;+ seq_id[i].resize(1);+ seq_id[i][0] = u_batch.all_seq_id;+ seq_id_arr[i] = seq_id[i].data();+ }- llama_kv_cache_update(&lctx);+ u_batch.n_seq_id = n_seq_id.data();+ u_batch.seq_id = seq_id_arr.data();+ }- // if we have enough unused cells before the current head ->- // better to start searching from the beginning of the cache, hoping to fill it- if (kv_self.head > kv_self.used + 2*n_tokens) {- kv_self.head = 0;- }+ // non-causal masks do not use the KV cache+ if (hparams.causal_attn) {+ llama_kv_cache_update(&lctx);- if (!llama_kv_cache_find_slot(kv_self, batch)) {- return 1;- }+ // if we have enough unused cells before the current head ->+ // better to start searching from the beginning of the cache, hoping to fill it+ if (kv_self.head > kv_self.used + 2*n_tokens) {+ kv_self.head = 0;+ }- // a heuristic, to avoid attending the full cache if it is not yet utilized- // after enough generations, the benefit from this heuristic disappears- // if we start defragmenting the cache, the benefit from this will be more important- kv_self.n = std::min((int32_t) cparams.n_ctx, std::max(32, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32)));- //kv_self.n = llama_kv_cache_cell_max(kv_self);+ if (!llama_kv_cache_find_slot(kv_self, u_batch)) {+ return 1;+ }- //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head);+ if (!kv_self.recurrent) {+ // a heuristic, to avoid attending the full cache if it is not yet utilized+ // after enough generations, the benefit from this heuristic disappears+ // if we start defragmenting the cache, the benefit from this will be more important+ kv_self.n = std::min(kv_self.size, std::max(32u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32)));+ //kv_self.n = llama_kv_cache_cell_max(kv_self);+ }+ }- ggml_backend_sched_reset(lctx.sched);- ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data);+ //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head);- ggml_cgraph * gf = llama_build_graph(lctx, batch, false);+ ggml_backend_sched_reset(lctx.sched);+ ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data);- // the output is always the last tensor in the graph- struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1];- struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2];+ ggml_cgraph * gf = llama_build_graph(lctx, u_batch, false);- if (strcmp(res->name, "result_output") == 0) {- // the embeddings could be the second to last tensor, or the third to last tensor- if (strcmp(embeddings->name, "result_norm") != 0) {- embeddings = gf->nodes[gf->n_nodes - 3];- GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0);- }- } else if (strcmp(res->name, "result_embd") == 0) {- embeddings = res;- res = nullptr;- } else {- GGML_ASSERT(false);- }+ // the output is always the last tensor in the graph+ struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1];+ struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 2];- // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);+ if (!hparams.causal_attn) {+ res = nullptr; // do not extract logits for embedding models such as BERT- // for big prompts, if BLAS is enabled, it is better to use only one thread- // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance- // TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well- // we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering- // with the BLAS calls. need a better solution- // MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is- // being processed then Accelerate/BLAS will not be involved, so capping would limit performance.- if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) {- n_threads = std::min(4, n_threads);- }+ // token or sequence embeddings+ embd = gf->nodes[gf->n_nodes - 1];- llama_set_inputs(lctx, batch);+ GGML_ASSERT(strcmp(embd->name, "result_embd") == 0 || strcmp(embd->name, "result_embd_pooled") == 0);+ } else {+ if (strcmp(res->name, "result_output") == 0) {+ // the token embeddings could be the second to last tensor, or the third to last tensor+ if (strcmp(embd->name, "result_norm") != 0) {+ embd = gf->nodes[gf->n_nodes - 3];+ GGML_ASSERT(strcmp(embd->name, "result_norm") == 0);+ }+ } else {+ GGML_ASSERT(false && "missing result_output tensor");+ }+ }+ // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);- llama_graph_compute(lctx, gf, n_threads);+ // for big prompts, if BLAS is enabled, it is better to use only one thread+ // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance+ // TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well+ // we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering+ // with the BLAS calls. need a better solution+ // MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is+ // being processed then Accelerate/BLAS will not be involved, so capping would limit performance.+ if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) {+ n_threads = std::min(4, n_threads);+ }- // update the kv ring buffer- {- kv_self.head += n_tokens;+ ggml_backend_sched_alloc_graph(lctx.sched, gf);- // Ensure kv cache head points to a valid index.- if (kv_self.head >= kv_self.size) {- kv_self.head = 0;- }- }+ llama_set_inputs(lctx, u_batch);- // decide if we need to defrag the kv cache- if (cparams.defrag_thold >= 0.0f) {- const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used + n_tokens)/float(kv_self.n) : 0.0f;+ llama_graph_compute(lctx, gf, n_threads);- // queue defragmentation for next llama_kv_cache_update- if (fragmentation > cparams.defrag_thold) {- //LLAMA_LOG_INFO("fragmentation: %.2f\n", fragmentation);+ // update the kv ring buffer+ {+ kv_self.head += n_tokens;- llama_kv_cache_defrag(kv_self);+ // Ensure kv cache head points to a valid index.+ if (kv_self.head >= kv_self.size) {+ kv_self.head = 0;+ }}- }#ifdef GGML_PERF- // print timing information per ggml operation (for debugging purposes)- // requires GGML_PERF to be defined- ggml_graph_print(gf);+ // print timing information per ggml operation (for debugging purposes)+ // requires GGML_PERF to be defined+ ggml_graph_print(gf);#endif- // plot the computation graph in dot format (for debugging purposes)- //if (n_past%100 == 0) {- // ggml_graph_dump_dot(gf, NULL, "llama.dot");- //}-- // extract logits- // TODO: do not compute and extract logits if only embeddings are needed- // need to update the graphs to skip "result_output"- if (res) {- auto & logits_out = lctx.logits;-+ // plot the computation graph in dot format (for debugging purposes)+ //if (n_past%100 == 0) {+ // ggml_graph_dump_dot(gf, NULL, "llama.dot");+ //}++ // extract logits+ // TODO: do not compute and extract logits if only embeddings are needed+ // update the graphs to skip "result_output" if logits are not needed+ if (res) {+ ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(lctx.sched, res);+ GGML_ASSERT(backend_res != nullptr);+ if (u_batch.logits) {+ int32_t i_first = -1;+ for (uint32_t i = 0; i < n_tokens; i++) {+ if (u_batch.logits[i] && i_first == -1) {+ i_first = (int32_t) i;+ }+ if (u_batch.logits[i] == 0 || i == n_tokens - 1) {+ if (i_first != -1) {+ int i_last = u_batch.logits[i] == 0 ? i : i + 1;+ // extract logits for the range [i_first, i_last)+ // group the requests to minimize the number of calls to the backend+ ggml_backend_tensor_get_async(backend_res, res,+ logits_out + n_vocab*(cur_token + i_first),+ i_first*n_vocab*sizeof(float),+ (i_last - i_first)*n_vocab*sizeof(float));+ i_first = -1;+ }+ }#ifndef NDEBUG- auto & logits_valid = lctx.logits_valid;- logits_valid.clear();- logits_valid.resize(n_tokens);-- logits_out.clear();+ logits_valid[cur_token + i] = u_batch.logits[i] != 0;;#endif-- ggml_backend_t res_backend = ggml_backend_sched_get_node_backend(lctx.sched, res);- GGML_ASSERT(res_backend != nullptr);- if (batch.logits) {- logits_out.resize(n_vocab * n_tokens);- for (uint32_t i = 0; i < n_tokens; i++) {- if (batch.logits[i] == 0) {- continue;}- ggml_backend_tensor_get_async(res_backend, res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float));-#ifndef NDEBUG- logits_valid[i] = true;-#endif- }- } else if (lctx.logits_all) {- logits_out.resize(n_vocab * n_tokens);- ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float));+ } else if (lctx.logits_all) {+ ggml_backend_tensor_get_async(backend_res, res, logits_out + n_vocab*cur_token, 0, n_vocab*n_tokens*sizeof(float));#ifndef NDEBUG- std::fill(logits_valid.begin(), logits_valid.end(), true);+ std::fill(logits_valid.begin() + cur_token, logits_valid.begin() + cur_token + n_tokens, true);#endif- } else {- logits_out.resize(n_vocab);- ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float));+ } else {+ if (cur_token + n_tokens >= n_tokens_all) {+ ggml_backend_tensor_get_async(backend_res, res, logits_out, n_vocab*(n_tokens - 1)*sizeof(float), n_vocab*sizeof(float));#ifndef NDEBUG- logits_valid[0] = true;+ logits_valid[0] = true;#endif+ }+ }}- ggml_backend_synchronize(res_backend);- }- // extract embeddings- if (!lctx.embedding.empty()) {- auto & embedding_out = lctx.embedding;+ // extract embeddings+ if (cparams.embeddings && embd) {+ ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched, embd);+ GGML_ASSERT(backend_embd != nullptr);- const int64_t embd_pos = res ? n_embd * (n_tokens-1) : 0;- const int64_t embd_size = res ? n_embd : n_embd * n_tokens;+ switch (cparams.pooling_type) {+ case LLAMA_POOLING_TYPE_NONE:+ {+ // extract token embeddings+ auto & embd_out = lctx.embd;++ if (u_batch.logits) {+ //embd_out.resize(n_embd * n_tokens);+ for (uint32_t i = 0; i < n_tokens; i++) {+ if (u_batch.logits[i] == 0) {+ continue;+ }+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out + n_embd*(i + cur_token), (n_embd*i)*sizeof(float), n_embd*sizeof(float));+ }+ }+ } break;+ case LLAMA_POOLING_TYPE_CLS:+ case LLAMA_POOLING_TYPE_MEAN:+ {+ GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0);- embedding_out.resize(embd_size);- ggml_backend_t embeddings_backend = ggml_backend_sched_get_node_backend(lctx.sched, embeddings);- ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), embd_pos*sizeof(float), embd_size*sizeof(float));- ggml_backend_synchronize(embeddings_backend);- }+ // extract sequence embeddings+ auto & embd_seq_out = lctx.embd_seq;+ embd_seq_out.clear();- // measure the performance only for the single-token evals- if (n_tokens == 1) {- lctx.t_eval_us += ggml_time_us() - t_start_us;- lctx.n_eval++;- }- else if (n_tokens > 1) {- lctx.t_p_eval_us += ggml_time_us() - t_start_us;- lctx.n_p_eval += n_tokens;+ for (uint32_t i = 0; i < n_tokens; i++) {+ const llama_seq_id seq_id = u_batch.seq_id[i][0];+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {+ continue;+ }+ embd_seq_out[seq_id].resize(n_embd);+ ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));+ }+ } break;+ case LLAMA_POOLING_TYPE_UNSPECIFIED:+ {+ GGML_ASSERT(false && "unknown pooling type");+ } break;+ }+ }}- // get a more accurate load time, upon first eval- // TODO: fix this- if (!lctx.has_evaluated_once) {- lctx.t_load_us = ggml_time_us() - lctx.t_start_us;- lctx.has_evaluated_once = true;+ // wait for the computation to finish (automatically done when obtaining the model output)+ //llama_synchronize(&lctx);++ // decide if we need to defrag the kv cache+ if (cparams.causal_attn && cparams.defrag_thold >= 0.0f) {+ const float fragmentation = kv_self.n >= 128 ? 1.0f - float(kv_self.used)/float(kv_self.n) : 0.0f;++ // queue defragmentation for next llama_kv_cache_update+ if (fragmentation > cparams.defrag_thold) {+ //LLAMA_LOG_INFO("fragmentation: %.2f\n", fragmentation);++ llama_kv_cache_defrag(kv_self);+ }}return 0;}+// find holes from the beginning of the KV cache and fill them by moving data from the end of the cachestatic void llama_kv_cache_defrag_internal(struct llama_context & lctx) {auto & kv_self = lctx.kv_self;@@ -7927,6 +9125,11 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {// number of cells moveduint32_t n_moves = 0;+ // each move requires 6*n_layer tensors (see build_defrag)+ // - source view, destination view, copy operation+ // - x2 for keys and values+ const uint32_t max_moves = LLAMA_MAX_NODES/(6*n_layer);+// determine which KV cells to move where//// cell i moves to ids[i]@@ -7953,15 +9156,6 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {nh++;}- // each move requires 6*n_layer tensors (see build_defrag)- // - source view, destination view, copy operation- // - x2 for keys and values- //- if (6*(n_moves + nh)*n_layer >= LLAMA_MAX_NODES) {- // the graph is too big, we cannot move more cells- break;- }-uint32_t nf = 0;uint32_t is = n_kv - 1;@@ -7991,11 +9185,19 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {// are we moving a continuous block of memory?bool cont = false;+ // should we stop searching for the next move?+ bool stop = false;+// go back and move the nf cells to the holefor (; i1 < n_kv; ++i1) {auto & cell1 = kv_self.cells[i1];if (cell1.is_empty() || ids[i1] != n_kv) {+ if (n_moves == max_moves) {+ stop = true;+ break;+ }+cont = false;continue;}@@ -8022,6 +9224,10 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {}}+ if (stop || n_moves == max_moves) {+ break;+ }+//LLAMA_LOG_INFO("(tmp log) KV defrag: move [%u, %u) to [%u, %u)\n", is, i1 + 1, i0, i0 + nh);i0 += nh - 1;@@ -8108,6 +9314,8 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {#else// ggml_graph defrag+ ggml_backend_sched_reset(lctx.sched);+ggml_cgraph * gf = llama_build_graph_defrag(lctx, ids);llama_graph_compute(lctx, gf, lctx.cparams.n_threads);@@ -8119,14 +9327,22 @@ static void llama_kv_cache_defrag_internal(struct llama_context & lctx) {}static void llama_kv_cache_update_internal(struct llama_context & lctx) {+ bool need_reserve = false;+// apply K-shift if neededif (lctx.model.hparams.rope_type != LLAMA_ROPE_TYPE_NONE && lctx.kv_self.has_shift) {- llama_set_k_shift(lctx);-{+ ggml_backend_sched_reset(lctx.sched);+ggml_cgraph * gf = llama_build_graph_k_shift(lctx);+ ggml_backend_sched_alloc_graph(lctx.sched, gf);++ llama_set_k_shift(lctx);+llama_graph_compute(lctx, gf, lctx.cparams.n_threads);++ need_reserve = true;}{@@ -8140,12 +9356,56 @@ static void llama_kv_cache_update_internal(struct llama_context & lctx) {}}+ if (lctx.kv_self.recurrent && lctx.kv_self.do_copy) {+ {+ ggml_backend_sched_reset(lctx.sched);++ ggml_cgraph * gf = llama_build_graph_s_copy(lctx);++ ggml_backend_sched_alloc_graph(lctx.sched, gf);++ llama_set_s_copy(lctx);++ llama_graph_compute(lctx, gf, lctx.cparams.n_threads);++ need_reserve = true;+ }++ {+ auto & kv_self = lctx.kv_self;++ kv_self.do_copy = false;++ for (uint32_t i = 0; i < kv_self.size; ++i) {+ kv_self.cells[i].src = i;+ }+ }+ }+// defragment the KV cache if neededif (lctx.kv_self.do_defrag) {llama_kv_cache_defrag_internal(lctx);+ need_reserve = true;+lctx.kv_self.do_defrag = false;}++ // reserve a worst case graph again+ if (need_reserve) {+ // TODO: extract to a function+ // build worst-case graph+ int n_tokens = (int)std::min(lctx.cparams.n_ctx, lctx.cparams.n_ubatch);+ int n_past = lctx.cparams.n_ctx - n_tokens;+ llama_token token = llama_token_bos(&lctx.model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph+ ggml_cgraph * gf = llama_build_graph(lctx, llama_batch_get_one(&token, n_tokens, n_past, 0), true);++ // initialize scheduler with the worst-case graph+ ggml_backend_sched_reset(lctx.sched);+ if (!ggml_backend_sched_reserve(lctx.sched, gf)) {+ LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__);+ }+ }}//@@ -8157,46 +9417,53 @@ static enum llama_vocab_type llama_vocab_get_type(const llama_vocab & vocab) {}static bool llama_is_normal_token(const llama_vocab & vocab, llama_token id) {+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_NORMAL;}static bool llama_is_unknown_token(const llama_vocab & vocab, llama_token id) {+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_UNKNOWN;}static bool llama_is_control_token(const llama_vocab & vocab, llama_token id) {+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_CONTROL;}static bool llama_is_byte_token(const llama_vocab & vocab, llama_token id) {+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_BYTE;}static bool llama_is_user_defined_token(const llama_vocab& vocab, llama_token id) {+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);return vocab.id_to_token[id].type == LLAMA_TOKEN_TYPE_USER_DEFINED;}static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {+ GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);GGML_ASSERT(llama_is_byte_token(vocab, id));const auto& token_data = vocab.id_to_token.at(id);switch (llama_vocab_get_type(vocab)) {- case LLAMA_VOCAB_TYPE_SPM: {- auto buf = token_data.text.substr(3, 2);- return strtol(buf.c_str(), NULL, 16);- }- case LLAMA_VOCAB_TYPE_BPE: {- GGML_ASSERT(false);- return unicode_to_bytes_bpe(token_data.text);- }- case LLAMA_VOCAB_TYPE_WPM: {- GGML_ASSERT(false);- }- default:- GGML_ASSERT(false);+ case LLAMA_VOCAB_TYPE_SPM: {+ auto buf = token_data.text.substr(3, 2);+ return strtol(buf.c_str(), NULL, 16);+ }+ case LLAMA_VOCAB_TYPE_BPE: {+ GGML_ASSERT(false);+ return unicode_utf8_to_byte(token_data.text);+ }+ case LLAMA_VOCAB_TYPE_WPM: {+ GGML_ASSERT(false);+ }+ default:+ GGML_ASSERT(false);}}static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {+ GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);static const char * hex = "0123456789ABCDEF";switch (llama_vocab_get_type(vocab)) {case LLAMA_VOCAB_TYPE_SPM: {@@ -8211,7 +9478,7 @@ static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {}case LLAMA_VOCAB_TYPE_WPM:case LLAMA_VOCAB_TYPE_BPE: {- return vocab.token_to_id.at(bytes_to_unicode_bpe(ch));+ return vocab.token_to_id.at(unicode_byte_to_utf8(ch));}default:GGML_ASSERT(false);@@ -8551,9 +9818,9 @@ private:bpe_words.reserve(text.size());bpe_encoded_words.reserve(text.size());- auto cps = codepoints_from_utf8(text);- for (size_t i = 0; i < cps.size(); ++i)- text_utf.emplace_back(codepoint_to_utf8(cps[i]));+ const auto cpts = unicode_cpts_from_utf8(text);+ for (size_t i = 0; i < cpts.size(); ++i)+ text_utf.emplace_back(unicode_cpt_to_utf8(cpts[i]));for (int i = 0; i < (int)text_utf.size(); i++) {const std::string & utf_char = text_utf[i];@@ -8603,40 +9870,40 @@ private:}if (!split_condition && !collecting) {- if (codepoint_type(utf_char) == CODEPOINT_TYPE_LETTER || (!token.size() && utf_char == " " && codepoint_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) {+ if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) {collecting_letter = true;collecting = true;}- else if (codepoint_type(utf_char) == CODEPOINT_TYPE_DIGIT || (!token.size() && utf_char == " " && codepoint_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {collecting_numeric = true;collecting = true;}else if (- ((codepoint_type(utf_char) != CODEPOINT_TYPE_LETTER && codepoint_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (codepoint_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) ||- (!token.size() && utf_char == " " && codepoint_type(utf_char_next) != CODEPOINT_TYPE_LETTER && codepoint_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && codepoint_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE)+ ((unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) && (unicode_cpt_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) ||+ (!token.size() && utf_char == " " && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_DIGIT && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE)) {collecting_special = true;collecting = true;}- else if (codepoint_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && codepoint_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) {+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) {collecting_whitespace_lookahead = true;collecting = true;}- else if (codepoint_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) {+ else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) {split_condition = true;}}else if (!split_condition && collecting) {- if (collecting_letter && codepoint_type(utf_char) != CODEPOINT_TYPE_LETTER) {+ if (collecting_letter && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER) {split_condition = true;}- else if (collecting_numeric && codepoint_type(utf_char) != CODEPOINT_TYPE_DIGIT) {+ else if (collecting_numeric && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_DIGIT) {split_condition = true;}- else if (collecting_special && (codepoint_type(utf_char) == CODEPOINT_TYPE_LETTER || codepoint_type(utf_char) == CODEPOINT_TYPE_DIGIT || codepoint_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) {+ else if (collecting_special && (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_DIGIT || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) {split_condition = true;}- else if (collecting_whitespace_lookahead && (codepoint_type(utf_char_next) == CODEPOINT_TYPE_LETTER || codepoint_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {+ else if (collecting_whitespace_lookahead && (unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_DIGIT)) {split_condition = true;}}@@ -8665,7 +9932,7 @@ private:for (std::string & word : bpe_words) {std::string encoded_token = "";for (char & c : word) {- encoded_token += bytes_to_unicode_bpe(c);+ encoded_token += unicode_byte_to_utf8(c);}bpe_encoded_words.emplace_back(encoded_token);}@@ -8739,45 +10006,41 @@ struct llm_tokenizer_wpm {}std::vector<std::string> preprocess(const std::string & text) {- std::string ori_str = normalize(text);- uint64_t ori_size = ori_str.size();+ std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));- // single punct / single symbol / single digit- // baseline: add whitespace on the left and right of punct and chinese characters- std::vector<std::string> words;+ // strip accents, strip control, uniformize whitespace,+ // to lowercase, pad chinese characters, pad punctuationstd::string new_str = "";- uint64_t i = 0;- while (i < ori_size) {- int utf_char_len = utf8_len(ori_str[i]);- if ((utf_char_len == 1) && ispunct(ori_str[i])) {- new_str += " ";- new_str += ori_str[i];- new_str += " ";- i += 1;+ for (uint32_t code : cpts_nfd) {+ int type = unicode_cpt_type(code);+ if (type == CODEPOINT_TYPE_ACCENT_MARK || type == CODEPOINT_TYPE_CONTROL) {+ continue;}- else if ((utf_char_len == 3) && is_chinese_char(ori_str.substr(i, 3))) {+ code = to_lower(code);+ if (type == CODEPOINT_TYPE_WHITESPACE) {+ code = ' ';+ }+ std::string s = unicode_cpt_to_utf8(code);+ if (type == CODEPOINT_TYPE_PUNCTUATION || is_ascii_punct(code) || is_chinese_char(code)) {new_str += " ";- new_str += ori_str.substr(i, 3);+ new_str += s;new_str += " ";- i += 3;- }- else {- new_str += ori_str[i];- i += 1;+ } else {+ new_str += s;}}// split by whitespaceuint64_t l = 0;uint64_t r = 0;+ std::vector<std::string> words;while (r < new_str.size()) {// if is whitespaceif (isspace(new_str[r])) {if (r > l) words.push_back(new_str.substr(l, (r - l)));l = r + 1;r = l;- }- else {+ } else {r += 1;}}@@ -8787,95 +10050,34 @@ struct llm_tokenizer_wpm {return words;}- std::string normalize(const std::string & text) {- // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98- std::string text2 = strip_accents(text);- for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i])) {- char c = text2[i];- if (c >= 'A' && c <= 'Z') {- text2[i] = c - 'A' + 'a';- }- }- return text2;- }-- bool is_chinese_char(const std::string & str) {- int len = str.length();- unsigned int codepoint = 0;- int num_bytes = 0;- int i = 0;- unsigned char ch = static_cast<unsigned char>(str[i]);- if (ch <= 0x7f) {- codepoint = ch;- num_bytes = 1;- } else if ((ch >> 5) == 0x06) {- codepoint = ch & 0x1f;- num_bytes = 2;- } else if ((ch >> 4) == 0x0e) {- codepoint = ch & 0x0f;- num_bytes = 3;- } else if ((ch >> 3) == 0x1e) {- codepoint = ch & 0x07;- num_bytes = 4;- }- for (int j = 1; j < num_bytes; ++j) {- if (i + j >= len) {- return false; // incomplete UTF-8 character- }- unsigned char next_ch = static_cast<unsigned char>(str[i + j]);- if ((next_ch >> 6) != 0x02) {- return false; // invalid trailing byte- }- codepoint = (codepoint << 6) | (next_ch & 0x3f);- }- if ((codepoint >= 0x4E00 && codepoint <= 0x9FFF) ||- (codepoint >= 0x3400 && codepoint <= 0x4DBF) ||- (codepoint >= 0x20000 && codepoint <= 0x2A6DF) ||- (codepoint >= 0x2A700 && codepoint <= 0x2B73F) ||- (codepoint >= 0x2B740 && codepoint <= 0x2B81F) ||- (codepoint >= 0x2B920 && codepoint <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920- (codepoint >= 0xF900 && codepoint <= 0xFAFF) ||- (codepoint >= 0x2F800 && codepoint <= 0x2FA1F) ||- (codepoint >= 0x3000 && codepoint <= 0x303F) ||- (codepoint >= 0xFF00 && codepoint <= 0xFFEF)) {- return true; // NOLINT+ uint32_t to_lower(uint32_t code) {+ static const std::locale locale("en_US.UTF-8");+#if defined(_WIN32)+ if (code > 0xFFFF) {+ return code;}- return false;+#endif+ return std::tolower(wchar_t(code), locale);}- std::string strip_accents(const std::string & input_string) {- std::string resultString;- std::map<std::string, char> accent_map = {- {"À", 'A'}, {"Á", 'A'}, {"Â", 'A'}, {"Ã", 'A'}, {"Ä", 'A'}, {"Å", 'A'},- {"à", 'a'}, {"á", 'a'}, {"â", 'a'}, {"ã", 'a'}, {"ä", 'a'}, {"å", 'a'},- {"È", 'E'}, {"É", 'E'}, {"Ê", 'E'}, {"Ë", 'E'}, {"è", 'e'}, {"é", 'e'},- {"ê", 'e'}, {"ë", 'e'}, {"Ì", 'I'}, {"Í", 'I'}, {"Î", 'I'}, {"Ï", 'I'},- {"ì", 'i'}, {"í", 'i'}, {"î", 'i'}, {"ï", 'i'}, {"Ò", 'O'}, {"Ó", 'O'},- {"Ô", 'O'}, {"Õ", 'O'}, {"Ö", 'O'}, {"ò", 'o'}, {"ó", 'o'}, {"ô", 'o'},- {"õ", 'o'}, {"ö", 'o'}, {"Ù", 'U'}, {"Ú", 'U'}, {"Û", 'U'}, {"Ü", 'U'},- {"ù", 'u'}, {"ú", 'u'}, {"û", 'u'}, {"ü", 'u'}, {"Ý", 'Y'}, {"ý", 'y'},- {"Ç", 'C'}, {"ç", 'c'}, {"Ñ", 'N'}, {"ñ", 'n'},- };-- for (size_t i = 0; i < input_string.length();) {- int len = utf8_len(input_string[i]);- std::string curChar = input_string.substr(i, len);- auto iter = accent_map.find(curChar);- if (iter != accent_map.end()) {- resultString += iter->second;- } else {- resultString += curChar;- }- i += len;- }-- return resultString;+ bool is_ascii_punct(uint32_t code) {+ return code < 256 && ispunct(code);}- static size_t utf8_len(char src) {- const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};- uint8_t highbits = static_cast<uint8_t>(src) >> 4;- return lookup[highbits];+ bool is_chinese_char(uint32_t cpt) {+ if ((cpt >= 0x4E00 && cpt <= 0x9FFF) ||+ (cpt >= 0x3400 && cpt <= 0x4DBF) ||+ (cpt >= 0x20000 && cpt <= 0x2A6DF) ||+ (cpt >= 0x2A700 && cpt <= 0x2B73F) ||+ (cpt >= 0x2B740 && cpt <= 0x2B81F) ||+ (cpt >= 0x2B920 && cpt <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920+ (cpt >= 0xF900 && cpt <= 0xFAFF) ||+ (cpt >= 0x2F800 && cpt <= 0x2FA1F) ||+ (cpt >= 0x3000 && cpt <= 0x303F) ||+ (cpt >= 0xFF00 && cpt <= 0xFFEF)) {+ return true; // NOLINT+ }+ return false;}const llama_vocab & vocab;@@ -9093,6 +10295,8 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &}}} break;+ case LLAMA_VOCAB_TYPE_NONE:+ GGML_ASSERT(false);}return output;@@ -9449,7 +10653,7 @@ struct llama_grammar * llama_grammar_init(// loop over alternates of start rule to build initial stacksstd::vector<std::vector<const llama_grammar_element *>> stacks;- pos = rules[start_rule_index];+ pos = vec_rules[start_rule_index].data();do {std::vector<const llama_grammar_element *> stack;if (!llama_grammar_is_end_of_sequence(pos)) {@@ -9911,10 +11115,6 @@ void llama_sample_temp(struct llama_context * ctx, llama_token_data_array * cand}}-void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) {- llama_sample_temp(ctx, candidates_p, temp);-}-void llama_sample_repetition_penalties(struct llama_context * ctx,llama_token_data_array * candidates,@@ -10041,38 +11241,6 @@ void llama_sample_apply_guidance(ctx->t_sample_us += ggml_time_us() - t_start_sample_us;}-void llama_sample_classifier_free_guidance(- struct llama_context * ctx,- llama_token_data_array * candidates,- struct llama_context * guidance_ctx,- float scale) {- GGML_ASSERT(ctx);- int64_t t_start_sample_us;-- t_start_sample_us = ggml_time_us();- const size_t n_vocab = llama_n_vocab(llama_get_model(ctx));-- GGML_ASSERT(n_vocab == candidates->size);- GGML_ASSERT(!candidates->sorted);-- std::vector<float> logits_base(n_vocab);- for (size_t i = 0; i < n_vocab; ++i) {- logits_base[i] = candidates->data[i].logit;- }-- float * logits_guidance = llama_get_logits(guidance_ctx);-- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;- llama_sample_apply_guidance(ctx, logits_base.data(), logits_guidance, scale);- t_start_sample_us = ggml_time_us();-- for (size_t i = 0; i < n_vocab; ++i) {- candidates->data[i].logit = logits_base[i];- }-- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;-}-llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) {GGML_ASSERT(ctx);@@ -10500,13 +11668,16 @@ struct quantize_state_internal {bool has_imatrix = false;+ // used to figure out if a model shares tok_embd with the output weight+ bool has_output = false;+quantize_state_internal(const llama_model & model, const llama_model_quantize_params * params): model(model), params(params){}};-static void llama_convert_tensor_internal(+static void llama_tensor_dequantize_internal(struct ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers,const size_t nelements, const int nthread) {@@ -10567,7 +11738,7 @@ static void llama_convert_tensor_internal(workers.clear();}-static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {+static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {const std::string name = ggml_get_name(tensor);// TODO: avoid hardcoded tensor names - use the TN_* constants@@ -10597,8 +11768,7 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty// for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings// with the quantization of the output tensor- if (name == tn(LLM_TENSOR_OUTPUT, "weight") ||- (LLM_TENSOR_NAMES.at(arch).find(LLM_TENSOR_OUTPUT) == LLM_TENSOR_NAMES.at(arch).end() && name == "token_embd.weight")) {+ if (name == tn(LLM_TENSOR_OUTPUT, "weight") || (!qs.has_output && name == tn(LLM_TENSOR_TOKEN_EMBD, "weight"))) {int nx = tensor->ne[0];if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) {new_type = GGML_TYPE_Q8_0;@@ -10847,41 +12017,76 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyreturn new_type;}+static size_t llama_tensor_quantize_internal(enum ggml_type new_type, const float * f32_data, void * new_data, const int chunk_size, int nrows, int n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {+ std::mutex mutex;+ int counter = 0;+ size_t new_size = 0;+ if (nthread < 2) {+ // single-thread+ return ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);+ }+ auto compute = [&mutex, &counter, &new_size, new_type, f32_data, new_data, chunk_size,+ nrows, n_per_row, imatrix]() {+ const int nrows_per_chunk = chunk_size / n_per_row;+ size_t local_size = 0;+ while (true) {+ std::unique_lock<std::mutex> lock(mutex);+ int first_row = counter; counter += nrows_per_chunk;+ if (first_row >= nrows) {+ if (local_size > 0) {+ new_size += local_size;+ }+ break;+ }+ lock.unlock();+ const int this_nrow = std::min(nrows - first_row, nrows_per_chunk);+ local_size += ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);+ }+ };+ for (int it = 0; it < nthread - 1; ++it) {+ workers.emplace_back(compute);+ }+ compute();+ for (auto & w : workers) { w.join(); }+ workers.clear();+ return new_size;+}+static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) {- ggml_type quantized_type;+ ggml_type default_type;llama_ftype ftype = params->ftype;switch (params->ftype) {- case LLAMA_FTYPE_MOSTLY_Q4_0: quantized_type = GGML_TYPE_Q4_0; break;- case LLAMA_FTYPE_MOSTLY_Q4_1: quantized_type = GGML_TYPE_Q4_1; break;- case LLAMA_FTYPE_MOSTLY_Q5_0: quantized_type = GGML_TYPE_Q5_0; break;- case LLAMA_FTYPE_MOSTLY_Q5_1: quantized_type = GGML_TYPE_Q5_1; break;- case LLAMA_FTYPE_MOSTLY_Q8_0: quantized_type = GGML_TYPE_Q8_0; break;- case LLAMA_FTYPE_MOSTLY_F16: quantized_type = GGML_TYPE_F16; break;- case LLAMA_FTYPE_ALL_F32: quantized_type = GGML_TYPE_F32; break;+ case LLAMA_FTYPE_MOSTLY_Q4_0: default_type = GGML_TYPE_Q4_0; break;+ case LLAMA_FTYPE_MOSTLY_Q4_1: default_type = GGML_TYPE_Q4_1; break;+ case LLAMA_FTYPE_MOSTLY_Q5_0: default_type = GGML_TYPE_Q5_0; break;+ case LLAMA_FTYPE_MOSTLY_Q5_1: default_type = GGML_TYPE_Q5_1; break;+ case LLAMA_FTYPE_MOSTLY_Q8_0: default_type = GGML_TYPE_Q8_0; break;+ case LLAMA_FTYPE_MOSTLY_F16: default_type = GGML_TYPE_F16; break;+ case LLAMA_FTYPE_ALL_F32: default_type = GGML_TYPE_F32; break;// K-quantscase LLAMA_FTYPE_MOSTLY_Q2_K_S:- case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break;- case LLAMA_FTYPE_MOSTLY_IQ3_XS: quantized_type = GGML_TYPE_IQ3_S; break;+ case LLAMA_FTYPE_MOSTLY_Q2_K: default_type = GGML_TYPE_Q2_K; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_XS: default_type = GGML_TYPE_IQ3_S; break;case LLAMA_FTYPE_MOSTLY_Q3_K_S:case LLAMA_FTYPE_MOSTLY_Q3_K_M:- case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break;+ case LLAMA_FTYPE_MOSTLY_Q3_K_L: default_type = GGML_TYPE_Q3_K; break;case LLAMA_FTYPE_MOSTLY_Q4_K_S:- case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break;+ case LLAMA_FTYPE_MOSTLY_Q4_K_M: default_type = GGML_TYPE_Q4_K; break;case LLAMA_FTYPE_MOSTLY_Q5_K_S:- case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break;- case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break;- case LLAMA_FTYPE_MOSTLY_IQ2_XXS: quantized_type = GGML_TYPE_IQ2_XXS; break;- case LLAMA_FTYPE_MOSTLY_IQ2_XS: quantized_type = GGML_TYPE_IQ2_XS; break;- case LLAMA_FTYPE_MOSTLY_IQ2_S: quantized_type = GGML_TYPE_IQ2_XS; break;- case LLAMA_FTYPE_MOSTLY_IQ2_M: quantized_type = GGML_TYPE_IQ2_S; break;- case LLAMA_FTYPE_MOSTLY_IQ3_XXS: quantized_type = GGML_TYPE_IQ3_XXS; break;- case LLAMA_FTYPE_MOSTLY_IQ1_S: quantized_type = GGML_TYPE_IQ1_S; break;- case LLAMA_FTYPE_MOSTLY_IQ4_NL: quantized_type = GGML_TYPE_IQ4_NL; break;- case LLAMA_FTYPE_MOSTLY_IQ4_XS: quantized_type = GGML_TYPE_IQ4_XS; break;- case LLAMA_FTYPE_MOSTLY_IQ3_S: quantized_type = GGML_TYPE_IQ3_S; break;- case LLAMA_FTYPE_MOSTLY_IQ3_M: quantized_type = GGML_TYPE_IQ3_S; break;+ case LLAMA_FTYPE_MOSTLY_Q5_K_M: default_type = GGML_TYPE_Q5_K; break;+ case LLAMA_FTYPE_MOSTLY_Q6_K: default_type = GGML_TYPE_Q6_K; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_XXS: default_type = GGML_TYPE_IQ2_XXS; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_XS: default_type = GGML_TYPE_IQ2_XS; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_S: default_type = GGML_TYPE_IQ2_XS; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_M: default_type = GGML_TYPE_IQ2_S; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_XXS: default_type = GGML_TYPE_IQ3_XXS; break;+ case LLAMA_FTYPE_MOSTLY_IQ1_S: default_type = GGML_TYPE_IQ1_S; break;+ case LLAMA_FTYPE_MOSTLY_IQ4_NL: default_type = GGML_TYPE_IQ4_NL; break;+ case LLAMA_FTYPE_MOSTLY_IQ4_XS: default_type = GGML_TYPE_IQ4_XS; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_S: default_type = GGML_TYPE_IQ3_S; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_M: default_type = GGML_TYPE_IQ3_S; break;default: throw std::runtime_error(format("invalid output file type %d\n", ftype));}@@ -10947,6 +12152,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const selse if (name.find("ffn_up") != std::string::npos) {++qs.n_ffn_up;}+ else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) {+ qs.has_output = true;+ }}if (qs.n_attention_wv != qs.n_ffn_down || (uint32_t)qs.n_attention_wv != model.hparams.n_layer) {LLAMA_LOG_WARN("%s ============ Strange model: n_attention_wv = %d, n_ffn_down = %d, hparams.n_layer = %d\n",@@ -10955,11 +12163,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const ssize_t total_size_org = 0;size_t total_size_new = 0;- std::vector<int64_t> hist_all(1 << 4, 0);std::vector<std::thread> workers;workers.reserve(nthread);- std::mutex mutex;int idx = 0;@@ -11011,26 +12217,36 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const squantize &= !params->only_copy;// do not quantize expert gating tensors- quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_FFN_GATE_INP, "weight");+ // NOTE: can't use LLM_TN here because the layer number is not known+ quantize &= name.find("ffn_gate_inp.weight") == std::string::npos;// do not quantize positional embeddings and token types (BERT)quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight");quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_TOKEN_TYPES, "weight");+ // do not quantize Mamba's small yet 2D weights+ // NOTE: can't use LLM_TN here because the layer number is not known+ quantize &= name.find("ssm_conv1d.weight") == std::string::npos;+ quantize &= name.find("ssm_x.weight") == std::string::npos;+ quantize &= name.find("ssm_dt.weight") == std::string::npos;+enum ggml_type new_type;void * new_data;size_t new_size;if (quantize) {- new_type = quantized_type;- if (!params->pure) {- new_type = get_k_quant_type(qs, new_type, tensor, ftype);+ new_type = default_type;++ // get more optimal quantization type based on the tensor shape, layer, etc.+ if (!params->pure && ggml_is_quantized(default_type)) {+ new_type = llama_tensor_get_type(qs, new_type, tensor, ftype);}// If we've decided to quantize to the same type the tensor is already// in then there's nothing to do.quantize = tensor->type != new_type;}+if (!quantize) {new_type = tensor->type;new_data = tensor->data;@@ -11072,18 +12288,17 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s} else if (ggml_is_quantized(tensor->type) && !params->allow_requantize) {throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor->type)));} else {- llama_convert_tensor_internal(tensor, f32_conv_buf, workers, nelements, nthread);+ llama_tensor_dequantize_internal(tensor, f32_conv_buf, workers, nelements, nthread);f32_data = (float *) f32_conv_buf.data();}- LLAMA_LOG_INFO("quantizing to %s .. ", ggml_type_name(new_type));+ LLAMA_LOG_INFO("converting to %s .. ", ggml_type_name(new_type));fflush(stdout);if (work.size() < nelements * 4) {work.resize(nelements * 4); // upper bound on size}new_data = work.data();- std::array<int64_t, 1 << 4> hist_cur = {};const int n_per_row = tensor->ne[0];const int nrows = nelements / n_per_row;@@ -11093,56 +12308,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sconst int nchunk = (nelements + chunk_size - 1)/chunk_size;const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1;- if (nthread_use < 2) {- new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, hist_cur.data(), imatrix);- } else {- int counter = 0;- new_size = 0;- auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, chunk_size,- nrows, n_per_row, imatrix]() {- std::array<int64_t, 1 << 4> local_hist = {};- const int nrows_per_chunk = chunk_size / n_per_row;- size_t local_size = 0;- while (true) {- std::unique_lock<std::mutex> lock(mutex);- int first_row = counter; counter += nrows_per_chunk;- if (first_row >= nrows) {- if (local_size > 0) {- for (int j=0; j<int(local_hist.size()); ++j) {- hist_cur[j] += local_hist[j];- }- new_size += local_size;- }- break;- }- lock.unlock();- const int this_nrow = std::min(nrows - first_row, nrows_per_chunk);- local_size += ggml_quantize_chunk(new_type, f32_data, new_data,- first_row * n_per_row, this_nrow, n_per_row, local_hist.data(), imatrix);- }- };- for (int it = 0; it < nthread_use - 1; ++it) {- workers.emplace_back(compute);- }- compute();- for (auto & w : workers) { w.join(); }- workers.clear();- }-- LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0);- int64_t tot_count = 0;- for (size_t i = 0; i < hist_cur.size(); i++) {- hist_all[i] += hist_cur[i];- tot_count += hist_cur[i];- }+ new_size = llama_tensor_quantize_internal(new_type, f32_data, new_data, chunk_size, nrows, n_per_row, imatrix, workers, nthread_use);- if (tot_count > 0) {- LLAMA_LOG_INFO(" | hist: ");- for (size_t i = 0; i < hist_cur.size(); i++) {- LLAMA_LOG_INFO("%5.3f ", hist_cur[i] / float(nelements));- }- }- LLAMA_LOG_INFO("\n");+ LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0);}total_size_org += ggml_nbytes(tensor);total_size_new += new_size;@@ -11171,24 +12339,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sLLAMA_LOG_INFO("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);LLAMA_LOG_INFO("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0);- // print histogram for all tensors- {- int64_t sum_all = 0;- for (size_t i = 0; i < hist_all.size(); i++) {- sum_all += hist_all[i];- }-- if (sum_all > 0) {- LLAMA_LOG_INFO("%s: hist: ", __func__);- for (size_t i = 0; i < hist_all.size(); i++) {- LLAMA_LOG_INFO("%5.3f ", hist_all[i] / float(sum_all));- }- LLAMA_LOG_INFO("\n");- }- }-if (qs.n_fallback > 0) {- LLAMA_LOG_WARN("%s: WARNING: %d of %d tensor(s) incompatible with k-quants and required fallback quantization\n",+ LLAMA_LOG_WARN("%s: WARNING: %d of %d tensor(s) required fallback quantization\n",__func__, qs.n_fallback, qs.n_k_quantized + qs.n_fallback);}}@@ -11500,10 +12652,13 @@ struct llama_context_params llama_context_default_params() {struct llama_context_params result = {/*.seed =*/ LLAMA_DEFAULT_SEED,/*.n_ctx =*/ 512,- /*.n_batch =*/ 512,+ /*.n_batch =*/ 2048,+ /*.n_ubatch =*/ 512,+ /*.n_seq_max =*/ 1,/*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default/*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS,/*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,+ /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED,/*.rope_freq_base =*/ 0.0f,/*.rope_freq_scale =*/ 0.0f,/*.yarn_ext_factor =*/ -1.0f,@@ -11516,11 +12671,11 @@ struct llama_context_params llama_context_default_params() {/*.cb_eval_user_data =*/ nullptr,/*.type_k =*/ GGML_TYPE_F16,/*.type_v =*/ GGML_TYPE_F16,- /*.mul_mat_q =*/ true,/*.logits_all =*/ false,- /*.embedding =*/ false,+ /*.embeddings =*/ false,/*.offload_kqv =*/ true,- /*.do_pooling =*/ true,+ /*.abort_callback =*/ nullptr,+ /*.abort_callback_data =*/ nullptr,};return result;@@ -11656,6 +12811,17 @@ struct llama_context * llama_new_context_with_model(struct llama_context_params params) {if (!model) {+ LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__);+ return nullptr;+ }++ if (params.n_batch == 0 && params.n_ubatch == 0) {+ LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__);+ return nullptr;+ }++ if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) {+ LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__);return nullptr;}@@ -11664,7 +12830,7 @@ struct llama_context * llama_new_context_with_model(const auto & hparams = model->hparams;auto & cparams = ctx->cparams;- cparams.n_batch = params.n_batch;+ // TODO: maybe add n_seq_max here toocparams.n_threads = params.n_threads;cparams.n_threads_batch = params.n_threads_batch;cparams.yarn_ext_factor = params.yarn_ext_factor;@@ -11672,14 +12838,19 @@ struct llama_context * llama_new_context_with_model(cparams.yarn_beta_fast = params.yarn_beta_fast;cparams.yarn_beta_slow = params.yarn_beta_slow;cparams.defrag_thold = params.defrag_thold;- cparams.mul_mat_q = params.mul_mat_q;+ cparams.embeddings = params.embeddings;cparams.offload_kqv = params.offload_kqv;- cparams.do_pooling = params.do_pooling;+ cparams.pooling_type = params.pooling_type;cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;+ // with causal attention, the batch size is limited by the context size+ cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;+ cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);++cparams.n_yarn_orig_ctx = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :hparams.n_yarn_orig_ctx != 0 ? hparams.n_yarn_orig_ctx :hparams.n_ctx_train;@@ -11700,19 +12871,44 @@ struct llama_context * llama_new_context_with_model(cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;}+ cparams.causal_attn = hparams.causal_attn;++ if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {+ if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {+ cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;+ } else {+ cparams.pooling_type = hparams.pooling_type;+ }+ }+if (params.seed == LLAMA_DEFAULT_SEED) {params.seed = _rand64();}LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);+ LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);+ LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);- ctx->rng = std::mt19937(params.seed);- ctx->logits_all = params.logits_all;+ ctx->abort_callback = params.abort_callback;+ ctx->abort_callback_data = params.abort_callback_data;++ ctx->rng = std::mt19937(params.seed);+ ctx->logits_all = params.logits_all;- const ggml_type type_k = params.type_k;- const ggml_type type_v = params.type_v;+ uint32_t kv_size = cparams.n_ctx;+ ggml_type type_k = params.type_k;+ ggml_type type_v = params.type_v;++ // Mamba only needs a constant number of KV cache cells per sequence+ if (model->arch == LLM_ARCH_MAMBA) {+ // Mamba needs at least as many KV cells as there are sequences kept at any time+ kv_size = std::max((uint32_t) 1, params.n_seq_max);+ // it's probably best to keep as much precision as possible for the states+ type_k = GGML_TYPE_F32; // required by ggml_ssm_conv for Mamba's conv_states+ type_v = GGML_TYPE_F32; // required by ggml_ssm_scan for Mamba's ssm_states+ }GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0);GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0);@@ -11730,27 +12926,25 @@ if (llamafile_has_metal()) {ctx->backends.push_back(ctx->backend_metal);}} else if (llamafile_has_cuda()) {- if (model->n_gpu_layers > 0) {+ if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {// with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used- if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {- ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu);+ ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu);+ if (backend == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu);+ llama_free(ctx);+ return nullptr;+ }+ ctx->backends.push_back(backend);+ } else {+ // LLAMA_SPLIT_MODE_LAYER requires a backend for each GPU+ for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) {+ ggml_backend_t backend = ggml_backend_cuda_init(device);if (backend == nullptr) {- LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu);+ LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, device);llama_free(ctx);return nullptr;}ctx->backends.push_back(backend);- } else {- // LLAMA_SPLIT_MODE_LAYER requires a backend for each GPU- for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) {- ggml_backend_t backend = ggml_backend_cuda_init(device);- if (backend == nullptr) {- LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, device);- llama_free(ctx);- return nullptr;- }- ctx->backends.push_back(backend);- }}}// #elif defined(GGML_USE_VULKAN)@@ -11767,13 +12961,30 @@ if (llamafile_has_metal()) {// }// #elif defined(GGML_USE_SYCL)// if (model->n_gpu_layers > 0) {-// ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu);-// if (backend == nullptr) {-// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d backend\n", __func__, model->main_gpu);-// llama_free(ctx);-// return nullptr;+// // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used+// if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {+// ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu);+// if (backend == nullptr) {+// int main_gpu_id = ggml_backend_sycl_get_device_id(model->main_gpu);+// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, main_gpu_id, model->main_gpu);+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// } else {+// // LLAMA_SPLIT_LAYER requires a backend for each GPU+// for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) {+// ggml_backend_t backend = ggml_backend_sycl_init(i);+// if (backend == nullptr) {+// int id_list[GGML_SYCL_MAX_DEVICES];+// ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES);+// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, id_list[i], i);+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// }// }-// ctx->backends.push_back(backend);// }// #elif defined(GGML_USE_KOMPUTE)// if (model->n_gpu_layers > 0) {@@ -11795,7 +13006,7 @@ if (llamafile_has_metal()) {}ctx->backends.push_back(ctx->backend_cpu);- if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, cparams.n_ctx, cparams.offload_kqv)) {+ if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v, kv_size, cparams.offload_kqv)) {LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__);llama_free(ctx);return nullptr;@@ -11819,45 +13030,31 @@ if (llamafile_has_metal()) {ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));}- // resized during inference, reserve maximum- ctx->logits.reserve(hparams.n_vocab*cparams.n_batch);-- if (params.embedding) {- ctx->embedding.resize(hparams.n_embd);- }-- // graph inputs+ // graph outputs buffer{- ggml_init_params init_params = {- /* .mem_size */ ggml_tensor_overhead()*8,- /* .mem_buffer */ nullptr,- /* .no_alloc */ true,- };- ctx->ctx_input = ggml_init(init_params);+ // resized during inference, reserve maximum+ ctx->logits_size = hparams.n_vocab*cparams.n_batch;+ ctx->embd_size = params.embeddings ? hparams.n_embd*cparams.n_batch : 0;- ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);- ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch);- ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);- ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch);- ctx->inp_KQ_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx);- ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx);- ctx->inp_mean = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_batch, cparams.n_batch);- ctx->inp_cls = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);+ const size_t buf_output_size = (ctx->logits_size + ctx->embd_size)*sizeof(float);++ ctx->buf_output = ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buf_output_size);+ if (ctx->buf_output == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to allocate logits buffer\n", __func__);+ llama_free(ctx);+ return nullptr;+ }+ ggml_backend_buffer_clear(ctx->buf_output, 0);- ggml_set_name(ctx->inp_tokens, "inp_tokens");- ggml_set_name(ctx->inp_embd, "inp_embd");- ggml_set_name(ctx->inp_pos, "inp_pos");- ggml_set_name(ctx->inp_KQ_mask, "inp_KQ_mask");- ggml_set_name(ctx->inp_KQ_pos, "inp_KQ_pos");- ggml_set_name(ctx->inp_K_shift, "inp_K_shift");- ggml_set_name(ctx->inp_mean, "inp_mean");- ggml_set_name(ctx->inp_cls, "inp_cls");- ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true));+ ctx->logits = (float *) ggml_backend_buffer_get_base(ctx->buf_output);+ if (params.embeddings) {+ ctx->embd = ctx->logits + ctx->logits_size;+ }- LLAMA_LOG_INFO("%s: %10s input buffer size = %8.2f MiB\n", __func__,- ggml_backend_buffer_name(ctx->buf_input),- ggml_backend_buffer_get_size(ctx->buf_input) / 1024.0 / 1024.0);+ LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,+ ggml_backend_buffer_name(ctx->buf_output),+ ggml_backend_buffer_get_size(ctx->buf_output) / 1024.0 / 1024.0);}// scheduler and compute buffers@@ -11876,10 +13073,21 @@ if (llamafile_has_metal()) {// buffer used to store the computation graph and the tensor meta datactx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead_custom(LLAMA_MAX_NODES, false));- ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES);+ // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary+ bool pipeline_parallel = llama_get_device_count() > 1 && model->n_gpu_layers > (int)model->hparams.n_layer && model->split_mode == LLAMA_SPLIT_MODE_LAYER;+#ifndef GGML_USE_CUBLAS+ // pipeline parallelism requires support for async compute and events+ // currently this is only implemented in the CUDA backend+ pipeline_parallel = false;+#endif+ ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES, pipeline_parallel);++ if (pipeline_parallel) {+ LLAMA_LOG_INFO("%s: pipeline parallelism enabled (n_copies=%d)\n", __func__, ggml_backend_sched_get_n_copies(ctx->sched));+ }// build worst-case graph- int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch);+ int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_ubatch);int n_past = cparams.n_ctx - n_tokens;llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graphggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0), true);@@ -11895,14 +13103,17 @@ if (llamafile_has_metal()) {ggml_backend_t backend = ctx->backends[i];ggml_backend_buffer_type_t buft = backend_buft[i];size_t size = ggml_backend_sched_get_buffer_size(ctx->sched, backend);- LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,- ggml_backend_buft_name(buft),- size / 1024.0 / 1024.0);+ if (size > 1) {+ LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,+ ggml_backend_buft_name(buft),+ size / 1024.0 / 1024.0);+ }}// note: the number of splits during measure is higher than during inference due to the kv shiftint n_splits = ggml_backend_sched_get_n_splits(ctx->sched);- LLAMA_LOG_INFO("%s: graph splits (measure): %d\n", __func__, n_splits);+ LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, gf->n_nodes);+ LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits);}}@@ -11939,6 +13150,14 @@ uint32_t llama_n_batch(const struct llama_context * ctx) {return ctx->cparams.n_batch;}+uint32_t llama_n_ubatch(const struct llama_context * ctx) {+ return ctx->cparams.n_ubatch;+}++uint32_t llama_n_seq_max(const struct llama_context * ctx) {+ return ctx->kv_self.size;+}+enum llama_vocab_type llama_vocab_type(const struct llama_model * model) {return model->vocab.type;}@@ -11952,6 +13171,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_MPT:case LLM_ARCH_REFACT:case LLM_ARCH_BLOOM:+ case LLM_ARCH_MAMBA:return LLAMA_ROPE_TYPE_NONE;// use what we call a normal RoPE, operating on pairs of consecutive head values@@ -11963,6 +13183,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_ORION:case LLM_ARCH_INTERNLM2:case LLM_ARCH_MINICPM:+ case LLM_ARCH_COMMAND_R:return LLAMA_ROPE_TYPE_NORM;// the pairs of head values are offset by n_rot/2@@ -11975,6 +13196,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_QWEN2:case LLM_ARCH_PHI2:case LLM_ARCH_GEMMA:+ case LLM_ARCH_STARCODER2:return LLAMA_ROPE_TYPE_NEOX;// all model arches should be listed explicitly here@@ -11987,7 +13209,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {}int32_t llama_n_vocab(const struct llama_model * model) {- return model->vocab.id_to_token.size();+ return model->hparams.n_vocab;}int32_t llama_n_ctx_train(const struct llama_model * model) {@@ -11998,6 +13220,10 @@ int32_t llama_n_embd(const struct llama_model * model) {return model->hparams.n_embd;}+int32_t llama_n_layer(const struct llama_model * model) {+ return model->hparams.n_layer;+}+float llama_rope_freq_scale_train(const struct llama_model * model) {return model->hparams.rope_freq_scale_train;}@@ -12088,28 +13314,109 @@ uint32_t llama_model_quantize(}}-int32_t llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, float scale, const char * path_base_model, int32_t n_threads) {+int32_t llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, float scale, const char * path_base_model, int32_t n_threads) {try {- return llama_apply_lora_from_file_internal(ctx->model, path_lora, scale, path_base_model, n_threads);+ return llama_apply_lora_from_file_internal(*model, path_lora, scale, path_base_model, n_threads);} catch (const std::exception & err) {LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what());return 1;}}-int32_t llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, float scale, const char * path_base_model, int32_t n_threads) {- try {- return llama_apply_lora_from_file_internal(*model, path_lora, scale, path_base_model, n_threads);- } catch (const std::exception & err) {- LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what());+static bool llama_control_vector_init(struct llama_control_vector & cvec, const llama_model & model) {+ GGML_ASSERT(cvec.tensors.empty());+ GGML_ASSERT(cvec.ctxs.empty());+ GGML_ASSERT(cvec.bufs.empty());++ // count layer buffer types+ std::map<ggml_backend_buffer_type_t, int> buft_layer_count;+ for (int64_t i = 0; i < model.hparams.n_layer; i++) {+ buft_layer_count[model.buft_layer[i].buft]++;+ }++ // allocate contexts+ std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;+ for (auto & it : buft_layer_count) {+ int n_layers = it.second;+ struct ggml_init_params params = {+ /*.mem_size =*/ n_layers * ggml_tensor_overhead(),+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ true,+ };+ ggml_context * ctx = ggml_init(params);+ if (!ctx) {+ LLAMA_LOG_ERROR("%s: failed to allocate context for control vector\n", __func__);+ return 1;+ }+ ctx_map[it.first] = ctx;+ }++ // make tensors+ cvec.tensors.push_back(nullptr); // there's never a tensor for layer 0+ for (size_t il = 1; il < model.hparams.n_layer; il++) {+ struct ggml_context * ctx = ctx_map.at(model.buft_layer[il].buft);+ ggml_tensor * tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, model.hparams.n_embd);+ cvec.tensors.push_back(tensor);+ }++ // allocate tensors / buffers and zero+ for (auto it : ctx_map) {+ ggml_backend_buffer_type_t buft = it.first;+ ggml_context * ctx = it.second;+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);+ if (!buf) {+ LLAMA_LOG_ERROR("%s: failed to allocate buffer for control vector\n", __func__);+ return false;+ }+ ggml_backend_buffer_clear(buf, 0);+ cvec.ctxs.push_back(ctx);+ cvec.bufs.push_back(buf);+ }++ return true;+}++int32_t llama_control_vector_apply(struct llama_context * lctx, const float * data, size_t len, int32_t n_embd, int32_t il_start, int32_t il_end) {+ const llama_model & model = lctx->model;+ llama_control_vector & cvec = lctx->cvec;++ if (data == nullptr) {+ // disable the current control vector (but leave allocated for later)+ cvec.layer_start = -1;+ cvec.layer_end = -1;+ return 0;+ }++ if (n_embd != (int) model.hparams.n_embd) {+ LLAMA_LOG_ERROR("%s: control vector n_embd does not match model\n", __func__);return 1;}++ if (cvec.tensors.empty()) {+ if (!llama_control_vector_init(cvec, model)) {+ return 1;+ }+ }++ cvec.layer_start = il_start;+ cvec.layer_end = il_end;++ for (size_t il = 1; il < model.hparams.n_layer; il++) {+ assert(cvec.tensors[il] != nullptr);++ const size_t off = n_embd * (il - 1); // buffer doesn't have data for layer 0, since it's never present+ if (off + n_embd <= len) {+ ggml_backend_tensor_set(cvec.tensors[il], data + off, 0, n_embd * ggml_element_size(cvec.tensors[il]));+ }+ }++ return 0;}-struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq) {+struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max) {struct llama_kv_cache_view result = {/*.n_cells = */ 0,- /*.n_max_seq = */ n_max_seq,+ /*.n_seq_max = */ n_seq_max,/*.token_count = */ 0,/*.used_cells = */ llama_get_kv_cache_used_cells(ctx),/*.max_contiguous = */ 0,@@ -12137,7 +13444,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kvoid * p = realloc(view->cells, sizeof(struct llama_kv_cache_view_cell) * view->n_cells);GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells");view->cells = (struct llama_kv_cache_view_cell *)p;- p = realloc(view->cells_sequences, sizeof(llama_seq_id) * view->n_max_seq * view->n_cells);+ p = realloc(view->cells_sequences, sizeof(llama_seq_id) * view->n_seq_max * view->n_cells);GGML_ASSERT(p != nullptr && "Failed to alloc kv_cache_view cells sequences");view->cells_sequences = (llama_seq_id *)p;}@@ -12151,7 +13458,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kuint32_t max_contig = 0;int32_t max_contig_idx = -1;- for (int32_t i = 0; i < int32_t(ctx->kv_self.size); i++, c_curr++, cs_curr += view->n_max_seq) {+ for (int32_t i = 0; i < int32_t(ctx->kv_self.size); i++, c_curr++, cs_curr += view->n_seq_max) {const size_t curr_size = kv_cells[i].seq_id.size();token_count += curr_size;c_curr->pos = kv_cells[i].pos + kv_cells[i].delta;@@ -12168,7 +13475,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kint seq_idx = 0;for (const llama_seq_id it : kv_cells[i].seq_id) {- if (seq_idx >= view->n_max_seq) {+ if (seq_idx >= view->n_seq_max) {break;}cs_curr[seq_idx] = it;@@ -12177,7 +13484,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kif (seq_idx != 0) {used_cells++;}- for (; seq_idx < view->n_max_seq; seq_idx++) {+ for (; seq_idx < view->n_seq_max; seq_idx++) {cs_curr[seq_idx] = -1;}}@@ -12213,8 +13520,8 @@ void llama_kv_cache_clear(struct llama_context * ctx) {llama_kv_cache_clear(ctx->kv_self);}-void llama_kv_cache_seq_rm(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) {- llama_kv_cache_seq_rm(ctx->kv_self, seq_id, p0, p1);+bool llama_kv_cache_seq_rm(struct llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) {+ return llama_kv_cache_seq_rm(ctx->kv_self, seq_id, p0, p1);}void llama_kv_cache_seq_cp(struct llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {@@ -12265,12 +13572,17 @@ size_t llama_get_state_size(const struct llama_context * ctx) {const size_t s_rng = LLAMA_MAX_RNG_STATE;const size_t s_logits_size = sizeof(size_t);// assume worst case for logits although only currently set ones are serialized- const size_t s_logits = ctx->logits.capacity() * sizeof(float);+ const size_t s_logits = ctx->logits_size * sizeof(float);const size_t s_embedding_size = sizeof(size_t);- const size_t s_embedding = ctx->embedding.size() * sizeof(float);- const size_t s_kv_size = sizeof(size_t);- const size_t s_kv_ntok = sizeof(int);+ const size_t s_embedding = ctx->embd_size * sizeof(float);+ const size_t s_kv_buf_size = sizeof(size_t);+ const size_t s_kv_head = sizeof(uint32_t);+ const size_t s_kv_size = sizeof(uint32_t);+ const size_t s_kv_used = sizeof(uint32_t);const size_t s_kv = ctx->kv_self.total_size();+ // TODO: assume the max is more than 1 seq_id per KV cell+ const size_t s_kv_cell = sizeof(llama_pos) + sizeof(size_t) + sizeof(llama_seq_id);+ const size_t s_kv_cells = ctx->kv_self.size * s_kv_cell;const size_t s_total = (+ s_rng_size@@ -12279,9 +13591,12 @@ size_t llama_get_state_size(const struct llama_context * ctx) {+ s_logits+ s_embedding_size+ s_embedding+ + s_kv_buf_size+ + s_kv_head+ s_kv_size- + s_kv_ntok+ + s_kv_used+ s_kv+ + s_kv_cells);return s_total;@@ -12357,23 +13672,23 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat// copy logits{- const size_t logits_size = ctx->logits.size();+ const size_t logits_size = ctx->logits_size;data_ctx->write(&logits_size, sizeof(logits_size));if (logits_size) {- data_ctx->write(ctx->logits.data(), logits_size * sizeof(float));+ data_ctx->write(ctx->logits, logits_size * sizeof(float));}}// copy embeddings{- const size_t embedding_size = ctx->embedding.size();+ const size_t embeddings_size = ctx->embd_size;- data_ctx->write(&embedding_size, sizeof(embedding_size));+ data_ctx->write(&embeddings_size, sizeof(embeddings_size));- if (embedding_size) {- data_ctx->write(ctx->embedding.data(), embedding_size * sizeof(float));+ if (embeddings_size) {+ data_ctx->write(ctx->embd, embeddings_size * sizeof(float));}}@@ -12381,15 +13696,13 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat{const auto & kv_self = ctx->kv_self;const auto & hparams = ctx->model.hparams;- const auto & cparams = ctx->cparams;const uint32_t n_layer = hparams.n_layer;- const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa();- const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa();- const uint32_t n_ctx = cparams.n_ctx;+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s();+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa() + hparams.n_embd_v_s();const size_t kv_buf_size = kv_self.total_size();- const uint32_t kv_head = kv_self.head;+ const uint32_t kv_head = llama_kv_cache_cell_max(kv_self);const uint32_t kv_size = kv_self.size;const uint32_t kv_used = kv_self.used;@@ -12407,9 +13720,20 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_datggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size());data_ctx->write(tmp_buf.data(), tmp_buf.size());+ if (kv_self.recurrent) {+ // v is contiguous for recurrent models+ // TODO: use other tensors for state models than k and v+ const size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head);++ tmp_buf.resize(v_size);+ ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), 0, tmp_buf.size());+ data_ctx->write(tmp_buf.data(), tmp_buf.size());+ continue;+ }+// v is not contiguous, copy row by rowconst size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head);- const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx);+ const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_size);tmp_buf.resize(v_row_size);for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) {@@ -12419,7 +13743,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_dat}}- for (uint32_t i = 0; i < kv_size; ++i) {+ for (uint32_t i = 0; i < kv_head; ++i) {const auto & cell = kv_self.cells[i];const llama_pos pos = cell.pos;@@ -12443,8 +13767,8 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) {}// Sets the state reading from the specified source address-size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {- uint8_t * inp = src;+size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) {+ const uint8_t * inp = src;// set rng{@@ -12453,7 +13777,7 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {GGML_ASSERT(rng_size <= LLAMA_MAX_RNG_STATE);- std::string rng_str((char *)inp, rng_size); inp += rng_size;+ std::string rng_str((const char *)inp, rng_size); inp += rng_size;std::istringstream rng_ss(rng_str);rng_ss >> ctx->rng;@@ -12467,27 +13791,25 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size);- GGML_ASSERT(ctx->logits.capacity() >= logits_size);+ GGML_ASSERT(ctx->logits_size >= logits_size);if (logits_size) {- ctx->logits.resize(logits_size);-- memcpy(ctx->logits.data(), inp, logits_size * sizeof(float));+ memcpy(ctx->logits, inp, logits_size * sizeof(float));inp += logits_size * sizeof(float);}}// set embeddings{- size_t embedding_size;+ size_t embeddings_size;- memcpy(&embedding_size, inp, sizeof(embedding_size)); inp += sizeof(embedding_size);+ memcpy(&embeddings_size, inp, sizeof(embeddings_size)); inp += sizeof(embeddings_size);- GGML_ASSERT(ctx->embedding.capacity() == embedding_size);+ GGML_ASSERT(ctx->embd_size == embeddings_size);- if (embedding_size) {- memcpy(ctx->embedding.data(), inp, embedding_size * sizeof(float));- inp += embedding_size * sizeof(float);+ if (embeddings_size) {+ memcpy(ctx->embd, inp, embeddings_size * sizeof(float));+ inp += embeddings_size * sizeof(float);}}@@ -12495,12 +13817,10 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {{const auto & kv_self = ctx->kv_self;const auto & hparams = ctx->model.hparams;- const auto & cparams = ctx->cparams;const uint32_t n_layer = hparams.n_layer;- const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa();- const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa();- const uint32_t n_ctx = cparams.n_ctx;+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa() + hparams.n_embd_k_s();+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa() + hparams.n_embd_v_s();size_t kv_buf_size;uint32_t kv_head;@@ -12521,9 +13841,19 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size);inp += k_size;+ if (kv_self.recurrent) {+ // v is contiguous for recurrent models+ // TODO: use other tensors for state models than k and v+ const size_t v_size = ggml_row_size(kv_self.v_l[il]->type, n_embd_v_gqa*kv_head);++ ggml_backend_tensor_set(kv_self.v_l[il], inp, 0, v_size);+ inp += v_size;+ continue;+ }+// v is not contiguous, copy row by rowconst size_t v_row_size = ggml_row_size(kv_self.v_l[il]->type, kv_head);- const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, n_ctx);+ const size_t v_row_stride = ggml_row_size(kv_self.v_l[il]->type, kv_size);for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) {ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*v_row_stride, v_row_size);@@ -12532,13 +13862,15 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {}}+ GGML_ASSERT(kv_self.size == kv_size);+ctx->kv_self.head = kv_head;ctx->kv_self.size = kv_size;ctx->kv_self.used = kv_used;ctx->kv_self.cells.resize(kv_size);- for (uint32_t i = 0; i < kv_size; ++i) {+ for (uint32_t i = 0; i < kv_head; ++i) {llama_pos pos;size_t seq_id_size;@@ -12554,6 +13886,11 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {ctx->kv_self.cells[i].seq_id.insert(seq_id);}}++ for (uint32_t i = kv_head; i < kv_size; ++i) {+ ctx->kv_self.cells[i].pos = -1;+ ctx->kv_self.cells[i].seq_id.clear();+ }}const size_t nread = inp - src;@@ -12646,41 +13983,18 @@ bool llama_save_session_file(struct llama_context * ctx, const char * path_sessireturn true;}-int llama_eval(- struct llama_context * ctx,- llama_token * tokens,- int32_t n_tokens,- int32_t n_past) {- llama_kv_cache_seq_rm(ctx->kv_self, -1, n_past, -1);-- const int ret = llama_decode_internal(*ctx, llama_batch_get_one(tokens, n_tokens, n_past, 0));- if (ret < 0) {- LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret);- }-- return ret;+void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch) {+ ctx->cparams.n_threads = n_threads;+ ctx->cparams.n_threads_batch = n_threads_batch;}-int llama_eval_embd(- struct llama_context * ctx,- float * embd,- int32_t n_tokens,- int32_t n_past) {- llama_kv_cache_seq_rm(ctx->kv_self, -1, n_past, -1);-- llama_batch batch = { n_tokens, nullptr, embd, nullptr, nullptr, nullptr, nullptr, n_past, 1, 0, };-- const int ret = llama_decode_internal(*ctx, batch);- if (ret < 0) {- LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret);- }-- return ret;+void llama_set_abort_callback(struct llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) {+ ctx->abort_callback = abort_callback;+ ctx->abort_callback_data = abort_callback_data;}-void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch) {- ctx->cparams.n_threads = n_threads;- ctx->cparams.n_threads_batch = n_threads_batch;+void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn) {+ ctx->cparams.causal_attn = causal_attn;}struct llama_batch llama_batch_get_one(@@ -12749,32 +14063,81 @@ int32_t llama_decode(return ret;}+void llama_synchronize(struct llama_context * ctx) {+ ggml_backend_sched_synchronize(ctx->sched);++ // FIXME: if multiple single tokens are evaluated without a synchronization,+ // the stats will be added to the prompt evaluation stats+ // this should only happen when using batch size 1 to evaluate a batch++ // add the evaluation to the stats+ if (ctx->n_queued_tokens == 1) {+ ctx->t_eval_us += ggml_time_us() - ctx->t_compute_start_us;+ ctx->n_eval++;+ } else if (ctx->n_queued_tokens > 1) {+ ctx->t_p_eval_us += ggml_time_us() - ctx->t_compute_start_us;+ ctx->n_p_eval += ctx->n_queued_tokens;+ }++ // get a more accurate load time, upon first eval+ if (ctx->n_queued_tokens > 0 && !ctx->has_evaluated_once) {+ ctx->t_load_us = ggml_time_us() - ctx->t_start_us;+ ctx->has_evaluated_once = true;+ }++ ctx->n_queued_tokens = 0;+ ctx->t_compute_start_us = 0;+}+float * llama_get_logits(struct llama_context * ctx) {- return ctx->logits.data();+ llama_synchronize(ctx);++ return ctx->logits;}float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) {assert(ctx->logits_valid.at(i));- return ctx->logits.data() + i*ctx->model.hparams.n_vocab;++ llama_synchronize(ctx);++ return ctx->logits + i*ctx->model.hparams.n_vocab;}float * llama_get_embeddings(struct llama_context * ctx) {- return ctx->embedding.data();+ llama_synchronize(ctx);++ return ctx->embd;}float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) {- return ctx->embedding.data() + i*ctx->model.hparams.n_embd;+ llama_synchronize(ctx);++ return ctx->embd + i*ctx->model.hparams.n_embd;+}++float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) {+ llama_synchronize(ctx);++ auto it = ctx->embd_seq.find(seq_id);+ if (it == ctx->embd_seq.end()) {+ return nullptr;+ }++ return it->second.data();}const char * llama_token_get_text(const struct llama_model * model, llama_token token) {+ GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE);return model->vocab.id_to_token[token].text.c_str();}float llama_token_get_score(const struct llama_model * model, llama_token token) {+ GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE);return model->vocab.id_to_token[token].score;}llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token) {+ GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE);return model->vocab.id_to_token[token].type;}@@ -12819,12 +14182,12 @@ int32_t llama_tokenize(const char * text,int32_t text_len,llama_token * tokens,- int32_t n_max_tokens,+ int32_t n_tokens_max,bool add_bos,bool special) {auto res = llama_tokenize_internal(model->vocab, std::string(text, text_len), add_bos, special);- if (n_max_tokens < (int) res.size()) {+ if (n_tokens_max < (int) res.size()) {// LLAMA_LOG_ERROR("%s: too many tokens\n", __func__);return -((int) res.size());}@@ -12838,9 +14201,9 @@ int32_t llama_tokenize(static std::string llama_decode_text(const std::string & text) {std::string decoded_text;- auto unicode_sequences = codepoints_from_utf8(text);- for (auto& unicode_sequence : unicode_sequences) {- decoded_text += unicode_to_bytes_bpe(codepoint_to_utf8(unicode_sequence));+ auto unicode_sequences = unicode_cpts_from_utf8(text);+ for (auto & unicode_sequence : unicode_sequences) {+ decoded_text += unicode_utf8_to_byte(unicode_cpt_to_utf8(unicode_sequence));}return decoded_text;@@ -12865,7 +14228,7 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token} else if (llama_is_user_defined_token(model->vocab, token)) {std::string result = model->vocab.id_to_token[token].text;if (length < (int) result.length()) {- return -result.length();+ return -(int) result.length();}memcpy(buf, result.c_str(), result.length());return result.length();@@ -12900,7 +14263,7 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token} else if (llama_is_user_defined_token(model->vocab, token)) {std::string result = model->vocab.id_to_token[token].text;if (length < (int) result.length()) {- return -result.length();+ return -(int) result.length();}memcpy(buf, result.c_str(), result.length());return result.length();@@ -12937,7 +14300,7 @@ static int32_t llama_chat_apply_template_internal(std::string & dest, bool add_ass) {// Taken from the research: https://github.com/ggerganov/llama.cpp/issues/5527std::stringstream ss;- if (tmpl.find("<|im_start|>") != std::string::npos) {+ if (tmpl == "chatml" || tmpl.find("<|im_start|>") != std::string::npos) {// chatml templatefor (auto message : chat) {ss << "<|im_start|>" << message->role << "\n" << message->content << "<|im_end|>\n";@@ -12945,7 +14308,7 @@ static int32_t llama_chat_apply_template_internal(if (add_ass) {ss << "<|im_start|>assistant\n";}- } else if (tmpl.find("[INST]") != std::string::npos) {+ } else if (tmpl == "llama2" || tmpl.find("[INST]") != std::string::npos) {// llama2 template and its variants// [variant] support system messagebool support_system_message = tmpl.find("<<SYS>>") != std::string::npos;@@ -12980,7 +14343,7 @@ static int32_t llama_chat_apply_template_internal(}}// llama2 templates seem to not care about "add_generation_prompt"- } else if (tmpl.find("<|user|>") != std::string::npos) {+ } else if (tmpl == "zephyr" || tmpl.find("<|user|>") != std::string::npos) {// zephyr templatefor (auto message : chat) {ss << "<|" << message->role << "|>" << "\n" << message->content << "<|endoftext|>\n";@@ -12988,7 +14351,7 @@ static int32_t llama_chat_apply_template_internal(if (add_ass) {ss << "<|assistant|>\n";}- } else if (tmpl.find("bos_token + message['role']") != std::string::npos) {+ } else if (tmpl == "monarch" || tmpl.find("bos_token + message['role']") != std::string::npos) {// mlabonne/AlphaMonarch-7B template (the <s> is included inside history)for (auto message : chat) {std::string bos = (message == chat.front()) ? "" : "<s>"; // skip BOS for first message@@ -12997,7 +14360,7 @@ static int32_t llama_chat_apply_template_internal(if (add_ass) {ss << "<s>assistant\n";}- } else if (tmpl.find("<start_of_turn>") != std::string::npos) {+ } else if (tmpl == "gemma" || tmpl.find("<start_of_turn>") != std::string::npos) {// google/gemma-7b-itstd::string system_prompt = "";for (auto message : chat) {@@ -13019,6 +14382,26 @@ static int32_t llama_chat_apply_template_internal(if (add_ass) {ss << "<start_of_turn>model\n";}+ } else if (tmpl == "orion" || tmpl.find("'\\n\\nAssistant: ' + eos_token") != std::string::npos) {+ // OrionStarAI/Orion-14B-Chat+ std::string system_prompt = "";+ for (auto message : chat) {+ std::string role(message->role);+ if (role == "system") {+ // there is no system message support, we will merge it with user prompt+ system_prompt = message->content;+ continue;+ } else if (role == "user") {+ ss << "Human: ";+ if (!system_prompt.empty()) {+ ss << system_prompt << "\n\n";+ system_prompt = "";+ }+ ss << message->content << "\n\nAssistant: </s>";+ } else {+ ss << message->content << "</s>";+ }+ }} else {// template not supportedreturn -1;@@ -13044,23 +14427,27 @@ LLAMA_API int32_t llama_chat_apply_template(int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), model_template.size());if (res < 0) {// worst case: there is no information about template, we will use chatml by default- curr_tmpl = "<|im_start|>"; // see llama_chat_apply_template_internal+ curr_tmpl = "chatml"; // see llama_chat_apply_template_internal} else {curr_tmpl = std::string(model_template.data(), model_template.size());}}+// format the chat to stringstd::vector<const llama_chat_message *> chat_vec;chat_vec.resize(n_msg);for (size_t i = 0; i < n_msg; i++) {chat_vec[i] = &chat[i];}+std::string formatted_chat;int32_t res = llama_chat_apply_template_internal(curr_tmpl, chat_vec, formatted_chat, add_ass);if (res < 0) {return res;}- strncpy(buf, formatted_chat.c_str(), length);+ if (buf && length > 0) {+ strncpy(buf, formatted_chat.c_str(), length);+ }return res;}diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 426bc47d..9d6f458d 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -61,9 +61,10 @@ extern "C" {typedef int32_t llama_seq_id;enum llama_vocab_type {- LLAMA_VOCAB_TYPE_SPM = 0, // SentencePiece- LLAMA_VOCAB_TYPE_BPE = 1, // Byte Pair Encoding- LLAMA_VOCAB_TYPE_WPM = 2, // WordPiece+ LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab+ LLAMA_VOCAB_TYPE_SPM = 1, // SentencePiece+ LLAMA_VOCAB_TYPE_BPE = 2, // Byte Pair Encoding+ LLAMA_VOCAB_TYPE_WPM = 3, // WordPiece};// note: these values should be synchronized with ggml_rope@@ -131,6 +132,7 @@ extern "C" {};enum llama_pooling_type {+ LLAMA_POOLING_TYPE_UNSPECIFIED = -1,LLAMA_POOLING_TYPE_NONE = 0,LLAMA_POOLING_TYPE_MEAN = 1,LLAMA_POOLING_TYPE_CLS = 2,@@ -164,7 +166,7 @@ extern "C" {// - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)// - pos : the positions of the respective token in the sequence// - seq_id : the sequence to which the respective token belongs- // - logits : if zero, the logits for the respective token will not be output+ // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output//typedef struct llama_batch {int32_t n_tokens;@@ -174,7 +176,7 @@ extern "C" {llama_pos * pos;int32_t * n_seq_id;llama_seq_id ** seq_id;- int8_t * logits;+ int8_t * logits; // TODO: rename this to "output"// NOTE: helpers for smooth API transition - can be deprecated in the future// for future-proof code, use the above fields instead and ignore everything below@@ -235,10 +237,15 @@ extern "C" {struct llama_context_params {uint32_t seed; // RNG seed, -1 for randomuint32_t n_ctx; // text context, 0 = from model- uint32_t n_batch; // prompt processing maximum batch size+ uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode+ uint32_t n_ubatch; // physical maximum batch size+ uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)uint32_t n_threads; // number of threads to use for generationuint32_t n_threads_batch; // number of threads to use for batch processing- int32_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`++ enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`+ enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id+ // (ignored if no pooling layer)// ref: https://github.com/ggerganov/llama.cpp/pull/2054float rope_freq_base; // RoPE base frequency, 0 = from model@@ -257,11 +264,15 @@ extern "C" {enum ggml_type type_v; // data type for V cache// Keep the booleans together to avoid misalignment during copy-by-value.- bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true)- bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)- bool embedding; // embedding mode only+ bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)+ bool embeddings; // if true, extract embeddings (together with logits)bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU- bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)++ // Abort callback+ // if it returns true, execution of llama_decode() will be aborted+ // currently works only with CPU execution+ ggml_abort_callback abort_callback;+ void * abort_callback_data;};// model quantization parameters@@ -271,7 +282,7 @@ extern "C" {bool allow_requantize; // allow quantizing non-f32/f16 tensorsbool quantize_output_tensor; // quantize output.weightbool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored- bool pure; // disable k-quant mixtures and quantize all tensors to the same type+ bool pure; // quantize all tensors to the default typevoid * imatrix; // pointer to importance matrix data} llama_model_quantize_params;@@ -366,13 +377,12 @@ extern "C" {LLAMA_API bool llama_supports_mlock (void);LLAMA_API bool llama_supports_gpu_offload(void);- LLAMA_API DEPRECATED(bool llama_mmap_supported (void), "use llama_supports_mmap() instead");- LLAMA_API DEPRECATED(bool llama_mlock_supported(void), "use llama_supports_mlock() instead");-LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx);LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);+ LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);+ LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model);LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model);@@ -380,6 +390,7 @@ extern "C" {LLAMA_API int32_t llama_n_vocab (const struct llama_model * model);LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model);LLAMA_API int32_t llama_n_embd (const struct llama_model * model);+ LLAMA_API int32_t llama_n_layer (const struct llama_model * model);// Get the model's RoPE frequency scaling factorLLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);@@ -425,20 +436,26 @@ extern "C" {// The model needs to be reloaded before applying a new adapter, otherwise the adapter// will be applied on top of the previous one// Returns 0 on success- LLAMA_API DEPRECATED(int32_t llama_apply_lora_from_file(- struct llama_context * ctx,- const char * path_lora,- float scale,- const char * path_base_model,- int32_t n_threads),- "use llama_model_apply_lora_from_file instead");-LLAMA_API int32_t llama_model_apply_lora_from_file(const struct llama_model * model,- const char * path_lora,- float scale,- const char * path_base_model,- int32_t n_threads);+ const char * path_lora,+ float scale,+ const char * path_base_model,+ int32_t n_threads);++ // Apply a loaded control vector to a llama_context, or if data is NULL, clear+ // the currently loaded vector.+ // n_embd should be the size of a single layer's control, and data should point+ // to an n_embd x n_layers buffer starting from layer 1.+ // il_start and il_end are the layer range the vector should apply to (both inclusive)+ // See llama_control_vector_load in common to load a control vector.+ LLAMA_API int32_t llama_control_vector_apply(+ struct llama_context * lctx,+ const float * data,+ size_t len,+ int32_t n_embd,+ int32_t il_start,+ int32_t il_end);//// KV cache@@ -459,7 +476,7 @@ extern "C" {// Maximum number of sequences that can exist in a cell. It's not an error// if there are more sequences in a cell than this value, however they will// not be visible in the view cells_sequences.- int32_t n_max_seq;+ int32_t n_seq_max;// Number of tokens in the cache. For example, if there are two populated// cells, the first with 1 sequence id in it and the second with 2 sequence@@ -479,12 +496,12 @@ extern "C" {// Information for an individual cell.struct llama_kv_cache_view_cell * cells;- // The sequences for each cell. There will be n_max_seq items per cell.+ // The sequences for each cell. There will be n_seq_max items per cell.llama_seq_id * cells_sequences;};// Create an empty KV cache view. (use only for debugging purposes)- LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq);+ LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max);// Free a KV cache view. (use only for debugging purposes)LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view * view);@@ -507,7 +524,7 @@ extern "C" {// seq_id < 0 : match any sequence// p0 < 0 : [0, p1]// p1 < 0 : [p0, inf)- LLAMA_API void llama_kv_cache_seq_rm(+ LLAMA_API bool llama_kv_cache_seq_rm(struct llama_context * ctx,llama_seq_id seq_id,llama_pos p0,@@ -588,7 +605,7 @@ extern "C" {// Returns the number of bytes readLLAMA_API size_t llama_set_state_data(struct llama_context * ctx,- uint8_t * src);+ const uint8_t * src);// Save/load session fileLLAMA_API bool llama_load_session_file(@@ -608,27 +625,6 @@ extern "C" {// Decoding//- // Run the llama inference to obtain the logits and probabilities for the next token(s).- // tokens + n_tokens is the provided batch of new tokens to process- // n_past is the number of tokens to use from previous eval calls- // Returns 0 on success- // DEPRECATED: use llama_decode() instead- LLAMA_API DEPRECATED(int llama_eval(- struct llama_context * ctx,- llama_token * tokens,- int32_t n_tokens,- int32_t n_past),- "use llama_decode() instead");-- // Same as llama_eval, but use float matrix input directly.- // DEPRECATED: use llama_decode() instead- LLAMA_API DEPRECATED(int llama_eval_embd(- struct llama_context * ctx,- float * embd,- int32_t n_tokens,- int32_t n_past),- "use llama_decode() instead");-// Return batch for single sequence of tokens starting at pos_0//// NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it@@ -667,7 +663,19 @@ extern "C" {// n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch);- // Token logits obtained from the last call to llama_eval()+ // Set whether to use causal attention or not+ // If set to true, the model will only attend to the past tokens+ LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);++ // Set abort callback+ LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);++ // Wait until all computations are finished+ // This is automatically done when using one of the functions below to obtain the computation results+ // and is not necessary to call it explicitly in most cases+ LLAMA_API void llama_synchronize(struct llama_context * ctx);++ // Token logits obtained from the last call to llama_decode()// The logits for the last token are stored in the last row// Logits for which llama_batch.logits[i] == 0 are undefined// Rows: n_tokens provided with llama_batch@@ -678,14 +686,20 @@ extern "C" {// llama_get_logits(ctx) + i*n_vocabLLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);- // Get the embeddings for the input- // shape: [n_embd] (1-dimensional)+ // Get all output token embeddings+ // shape: [n_tokens*n_embd] (1-dimensional)LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);- // Get the embeddings for the ith sequence+ // Get the embeddings for the ith token// llama_get_embeddings(ctx) + i*n_embd+ // shape: [n_embd] (1-dimensional)LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);+ // Get the embeddings for a sequence id+ // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE+ // shape: [n_embd] (1-dimensional)+ LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);+//// Vocab//@@ -719,7 +733,7 @@ extern "C" {/// @details Convert the provided text into tokens./// @param tokens The tokens pointer must be large enough to hold the resulting tokens.- /// @return Returns the number of tokens on success, no more than n_max_tokens+ /// @return Returns the number of tokens on success, no more than n_tokens_max/// @return Returns a negative number on failure - the number of tokens that would have been returned/// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext./// Does not insert a leading space.@@ -728,7 +742,7 @@ extern "C" {const char * text,int32_t text_len,llama_token * tokens,- int32_t n_max_tokens,+ int32_t n_tokens_max,bool add_bos,bool special);@@ -802,13 +816,6 @@ extern "C" {float * logits_guidance,float scale);- LLAMA_API DEPRECATED(void llama_sample_classifier_free_guidance(- struct llama_context * ctx,- llama_token_data_array * candidates,- struct llama_context * guidance_ctx,- float scale),- "use llama_sample_apply_guidance() instead");-/// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.LLAMA_API void llama_sample_softmax(struct llama_context * ctx,@@ -862,12 +869,6 @@ extern "C" {llama_token_data_array * candidates,float temp);- LLAMA_API DEPRECATED(void llama_sample_temperature(- struct llama_context * ctx,- llama_token_data_array * candidates,- float temp),- "use llama_sample_temp instead");-/// @details Apply constraints from grammarLLAMA_API void llama_sample_grammar(struct llama_context * ctx,diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex 1a71d9f1..543ebc28 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -113,6 +113,7 @@ static std::string format(const char * fmt, ...) {#define TN_LLAVA_PROJ "mm.%d.%s"#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"+#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s"#define TN_IMAGE_NEWLINE "model.image_newline"@@ -120,12 +121,14 @@ enum projector_type {PROJECTOR_TYPE_MLP,PROJECTOR_TYPE_MLP_NORM,PROJECTOR_TYPE_LDP,+ PROJECTOR_TYPE_LDPV2,PROJECTOR_TYPE_UNKNOWN,};static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {{ PROJECTOR_TYPE_MLP, "mlp" },{ PROJECTOR_TYPE_LDP, "ldp" },+ { PROJECTOR_TYPE_LDPV2, "ldpv2"},};@@ -469,6 +472,14 @@ struct clip_vision_model {struct ggml_tensor * mm_model_block_2_block_2_0_w;struct ggml_tensor * mm_model_block_2_block_2_1_w;struct ggml_tensor * mm_model_block_2_block_2_1_b;++ // MobileVLM_V2 projection+ struct ggml_tensor * mm_model_mlp_0_w;+ struct ggml_tensor * mm_model_mlp_0_b;+ struct ggml_tensor * mm_model_mlp_2_w;+ struct ggml_tensor * mm_model_mlp_2_b;+ struct ggml_tensor * mm_model_peg_0_w;+ struct ggml_tensor * mm_model_peg_0_b;};struct clip_ctx {@@ -491,7 +502,6 @@ struct clip_ctx {// memory buffers to evaluate the modelggml_backend_buffer_t params_buffer = NULL;- ggml_backend_buffer_t compute_buffer = NULL;ggml_backend_t backend = NULL;ggml_gallocr_t compute_alloc = NULL;@@ -802,6 +812,29 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32}embeddings = block_1;}+ else if (ctx->proj_type == PROJECTOR_TYPE_LDPV2)+ {+ int n_patch = 24;+ struct ggml_tensor * mlp_0 = ggml_mul_mat(ctx0, model.mm_model_mlp_0_w, embeddings);+ mlp_0 = ggml_add(ctx0, mlp_0, model.mm_model_mlp_0_b);+ mlp_0 = ggml_gelu(ctx0, mlp_0);+ struct ggml_tensor * mlp_2 = ggml_mul_mat(ctx0, model.mm_model_mlp_2_w, mlp_0);+ mlp_2 = ggml_add(ctx0, mlp_2, model.mm_model_mlp_2_b);+ // mlp_2 ne = [2048, 576, 1, 1]+ // // AVG Pool Layer 2*2, strides = 2+ mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 0, 2, 3));+ // mlp_2 ne = [576, 2048, 1, 1]+ mlp_2 = ggml_reshape_4d(ctx0, mlp_2, n_patch, n_patch, mlp_2->ne[1], mlp_2->ne[2]);+ // mlp_2 ne [24, 24, 2048, 1]+ mlp_2 = ggml_pool_2d(ctx0, mlp_2, GGML_OP_POOL_AVG, 2, 2, 2, 2, 0, 0);+ // weight ne = [3, 3, 2048, 1]+ struct ggml_tensor * peg_0 = ggml_conv_depthwise_2d(ctx0, model.mm_model_peg_0_w, mlp_2, 1, 1, 1, 1, 1, 1);+ peg_0 = ggml_add(ctx0, peg_0, mlp_2);+ peg_0 = ggml_cont(ctx0, ggml_permute(ctx0, peg_0, 1, 2, 0, 3));+ peg_0 = ggml_add(ctx0, peg_0, model.mm_model_peg_0_b);+ peg_0 = ggml_reshape_3d(ctx0, peg_0, peg_0->ne[0], peg_0->ne[1] * peg_0->ne[2], peg_0->ne[3]);+ embeddings = peg_0;+ }else {GGML_ASSERT(false);}@@ -1167,7 +1200,18 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight"));vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight"));vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias"));- } else {+ }+ else if (new_clip->proj_type == PROJECTOR_TYPE_LDPV2)+ {+ // MobilVLM_V2 projection+ vision_model.mm_model_mlp_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "weight"));+ vision_model.mm_model_mlp_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 0, "bias"));+ vision_model.mm_model_mlp_2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "weight"));+ vision_model.mm_model_mlp_2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 2, "bias"));+ vision_model.mm_model_peg_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "weight"));+ vision_model.mm_model_peg_0_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_PEG, 0, "bias"));+ }+ else {std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type];throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));}@@ -1224,16 +1268,16 @@ struct clip_image_f32 * clip_image_f32_init() {void clip_image_u8_free(struct clip_image_u8 * img) { delete img; }void clip_image_f32_free(struct clip_image_f32 * img) { delete img; }-void clip_image_u8_batch_free(struct clip_image_u8_batch & batch) {- if (batch.size > 0) {- delete[] batch.data;- batch.size = 0;+void clip_image_u8_batch_free(struct clip_image_u8_batch * batch) {+ if (batch->size > 0) {+ delete[] batch->data;+ batch->size = 0;}}-void clip_image_f32_batch_free(struct clip_image_f32_batch & batch) {- if (batch.size > 0) {- delete[] batch.data;- batch.size = 0;+void clip_image_f32_batch_free(struct clip_image_f32_batch * batch) {+ if (batch->size > 0) {+ delete[] batch->data;+ batch->size = 0;}}@@ -1486,7 +1530,7 @@ static std::vector<clip_image_u8*> divide_to_patches_u8(const clip_image_u8 & im// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector// res_imgs memory is being allocated here, previous allocations will be freed if found-bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs) {+bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch * res_imgs) {bool pad_to_square = true;if (!ctx->has_vision_encoder) {fprintf(stderr, "This gguf file seems to have no vision encoder\n");@@ -1498,11 +1542,11 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clipad_to_square = false;}// free the previous res_imgs if any set- if (res_imgs.size > 0) {+ if (res_imgs->size > 0) {clip_image_f32_batch_free(res_imgs);}- res_imgs.data = nullptr;- res_imgs.size = 0;+ res_imgs->data = nullptr;+ res_imgs->size = 0;// the logic below is to pad the shorter side to the longer side with a background color: rgb(122, 116, 104)// see https://github.com/haotian-liu/LLaVA/blob/e854a2bf85118c504f6f16bf5c3c7c92f8fa8c6b/llava/conversation.py#L113-L156@@ -1557,11 +1601,11 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clibicubic_resize(*img, *image_original_resize, params.image_size, params.image_size); // in python this is "shortest_edge", but all CLIP are squarepatches.insert(patches.begin(), image_original_resize);// clip_image_f32_batch_init(patches.size());- res_imgs.size = patches.size();- res_imgs.data = new clip_image_f32[res_imgs.size];+ res_imgs->size = patches.size();+ res_imgs->data = new clip_image_f32[res_imgs->size];int num=0;for (auto& patch : patches) {- normalize_image_u8_to_f32(patch, &res_imgs.data[num], ctx->image_mean, ctx->image_std);+ normalize_image_u8_to_f32(patch, &res_imgs->data[num], ctx->image_mean, ctx->image_std);num++;}@@ -1649,9 +1693,9 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, cli// }// res_imgs.push_back(res);- res_imgs.size = 1;- res_imgs.data = new clip_image_f32[res_imgs.size];- res_imgs.data[0] = *res;+ res_imgs->size = 1;+ res_imgs->data = new clip_image_f32[res_imgs->size];+ res_imgs->data[0] = *res;clip_image_f32_free(res);return true;@@ -1665,6 +1709,9 @@ void clip_free(clip_ctx * ctx) {ggml_free(ctx->ctx_data);gguf_free(ctx->ctx_gguf);+ ggml_backend_buffer_free(ctx->params_buffer);+ ggml_backend_free(ctx->backend);+ ggml_gallocr_free(ctx->compute_alloc);delete ctx;}@@ -1706,7 +1753,7 @@ int clip_n_patches(const struct clip_ctx * ctx) {bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec) {if (!ctx->has_vision_encoder) {- fprintf(stderr, "This gguf file seems to have no vision encoder\n");+ printf("This gguf file seems to have no vision encoder\n");return false;}@@ -1718,7 +1765,7 @@ bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f3bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs, float * vec) {if (!ctx->has_vision_encoder) {- fprintf(stderr, "This gguf file seems to have no vision encoder\n");+ printf("This gguf file seems to have no vision encoder\n");return false;}@@ -1799,9 +1846,11 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_imaggml_backend_cpu_set_n_threads(ctx->backend, n_threads);}+#ifdef GGML_USE_METALif (ggml_backend_is_metal(ctx->backend)) {ggml_backend_metal_set_n_cb(ctx->backend, n_threads);}+#endifggml_backend_graph_compute(ctx->backend, gf);@@ -1852,7 +1901,6 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const istd::vector<uint8_t> work(512);std::vector<float> conv_buf(512);- std::vector<int64_t> hist_all(1 << 4, 0);size_t total_size_org = 0;size_t total_size_new = 0;@@ -1899,6 +1947,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const ibreak;default:printf("Please use an input file in f32 or f16\n");+ gguf_free(ctx_out);return false;}@@ -1907,48 +1956,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i}new_data = work.data();- std::vector<int64_t> hist_cur(1 << 4, 0);-- switch (new_type) {- case GGML_TYPE_Q4_0: {- new_size = ggml_quantize_q4_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q4_1: {- new_size = ggml_quantize_q4_1(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q5_0: {- new_size = ggml_quantize_q5_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q5_1: {- new_size = ggml_quantize_q5_1(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q8_0: {- new_size = ggml_quantize_q8_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q2_K: {- new_size = ggml_quantize_q2_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q3_K: {- new_size = ggml_quantize_q3_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q4_K: {- new_size = ggml_quantize_q4_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q5_K: {- new_size = ggml_quantize_q5_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- case GGML_TYPE_Q6_K: {- new_size = ggml_quantize_q6_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());- } break;- default: {- fprintf(stderr, "%s: unsupported quantization type %d\n", __func__, new_type);- return false;- }- }-- for (size_t j = 0; j < hist_cur.size(); ++j) {- hist_all[j] += hist_cur[j];- }+ new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, n_elms/cur->ne[0], cur->ne[0], nullptr);} else {new_type = cur->type;new_data = cur->data;@@ -1983,17 +1991,6 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i{printf("%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0);printf("%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0);-- int64_t sum_all = 0;- for (size_t i = 0; i < hist_all.size(); ++i) {- sum_all += hist_all[i];- }-- printf("%s: hist: ", __func__);- for (size_t i = 0; i < hist_all.size(); ++i) {- printf("%5.3f ", hist_all[i] / (float)sum_all);- }- printf("\n");}return true;@@ -2003,6 +2000,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {if (ctx->proj_type == PROJECTOR_TYPE_LDP) {return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0];}+ if (ctx->proj_type == PROJECTOR_TYPE_LDPV2) {+ return ctx->vision_model.mm_model_peg_0_b->ne[0];+ }if (ctx->proj_type == PROJECTOR_TYPE_MLP) {return ctx->vision_model.mm_2_b->ne[0];}diff --git a/llama.cpp/llava/clip.h b/llama.cpp/llava/clip.hindex e5bd5492..45bdad68 100644--- a/llama.cpp/llava/clip.h+++ b/llama.cpp/llava/clip.h@@ -60,8 +60,8 @@ CLIP_API struct clip_image_f32 * clip_image_f32_init();CLIP_API void clip_image_u8_free (struct clip_image_u8 * img);CLIP_API void clip_image_f32_free(struct clip_image_f32 * img);-CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch & batch);-CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch & batch);+CLIP_API void clip_image_u8_batch_free (struct clip_image_u8_batch * batch);+CLIP_API void clip_image_f32_batch_free(struct clip_image_f32_batch * batch);CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);@@ -69,7 +69,7 @@ CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img);/** preprocess img and store the result in res_imgs, pad_to_square may be overriden to false depending on model configuration */-CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs );+CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32_batch * res_imgs );CLIP_API struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);diff --git a/llama.cpp/llava/llava-quantize.cpp b/llama.cpp/llava/llava-quantize.cppindex 7b5350ad..27fd1d24 100644--- a/llama.cpp/llava/llava-quantize.cpp+++ b/llama.cpp/llava/llava-quantize.cpp@@ -5,6 +5,7 @@#include "llamafile/version.h"#include <stdio.h>#include <stdlib.h>+#include "llamafile/llamafile.h"#include <string.h>int main(int argc, char *argv[]) {diff --git a/llama.cpp/llava/llava.cpp b/llama.cpp/llava/llava.cppindex 0416bf46..455f9c02 100644--- a/llama.cpp/llava/llava.cpp+++ b/llama.cpp/llava/llava.cpp@@ -225,7 +225,7 @@ static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const cliclip_image_f32_batch img_res_v;img_res_v.size = 0;img_res_v.data = nullptr;- if (!clip_image_preprocess(ctx_clip, img, img_res_v)) {+ if (!clip_image_preprocess(ctx_clip, img, &img_res_v)) {fprintf(stderr, "%s: unable to preprocess image\n", __func__);delete[] img_res_v.data;return false;diff --git a/llama.cpp/llava/llava.h b/llama.cpp/llava/llava.hindex 69e819be..47b6784c 100644--- a/llama.cpp/llava/llava.h+++ b/llama.cpp/llava/llava.h@@ -32,9 +32,9 @@ struct llava_image_embed {};/** sanity check for clip <-> llava embed size match */-LLAVA_API bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip);+LLAVA_API bool llava_validate_embed_size(const struct llama_context * ctx_llama, const struct clip_ctx * ctx_clip);-LLAVA_API bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out);+LLAVA_API bool llava_image_embed_make_with_clip_img(struct clip_ctx * ctx_clip, int n_threads, const struct clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out);/** build an image embed from image file bytes */LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length);diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 58feeedf..3b8ebcec 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -547,6 +547,14 @@ int main(int argc, char ** argv) {std::vector<llama_token> embd;std::vector<llama_token> embd_guidance;+ // tokenized antiprompts+ std::vector<std::vector<llama_token>> antiprompt_ids;++ antiprompt_ids.reserve(params.antiprompt.size());+ for (const std::string & antiprompt : params.antiprompt) {+ antiprompt_ids.emplace_back(::llama_tokenize(ctx, antiprompt, false, true));+ }+struct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams);while ((n_remain != 0 && !is_antiprompt) || params.interactive) {@@ -805,6 +813,18 @@ int main(int argc, char ** argv) {}}+ // check for reverse prompt using special tokens+ llama_token last_token = llama_sampling_last(ctx_sampling);+ for (std::vector<llama_token> ids : antiprompt_ids) {+ if (ids.size() == 1 && last_token == ids[0]) {+ if (params.interactive) {+ is_interacting = true;+ }+ is_antiprompt = true;+ break;+ }+ }+if (is_antiprompt) {LOG("found antiprompt: %s\n", last_output.c_str());}@@ -894,6 +914,7 @@ int main(int argc, char ** argv) {const auto line_pfx = ::llama_tokenize(ctx, params.input_prefix, false, true);const auto line_inp = ::llama_tokenize(ctx, buffer, false, false);const auto line_sfx = ::llama_tokenize(ctx, params.input_suffix, false, true);+LOG("input tokens: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, line_inp).c_str());embd_inp.insert(embd_inp.end(), line_pfx.begin(), line_pfx.end());diff --git a/llama.cpp/perplexity/perplexity.cpp b/llama.cpp/perplexity/perplexity.cppindex 1b8b7213..0837e8b1 100644--- a/llama.cpp/perplexity/perplexity.cpp+++ b/llama.cpp/perplexity/perplexity.cpp@@ -440,7 +440,7 @@ static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params &return {tokens, std::exp(nll / count), logit_history, prob_history};}-static results_perplexity perplexity(llama_context * ctx, const gpt_params & params) {+static results_perplexity perplexity(llama_context * ctx, const gpt_params & params, const int32_t n_ctx) {if (params.ppl_stride > 0) {return perplexity_v2(ctx, params);}@@ -451,7 +451,6 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par// BOS tokens will be added for each chunk before evalconst bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));- const int n_ctx = llama_n_ctx(ctx);std::ofstream logits_stream;if (!params.logits_file.empty()) {@@ -497,13 +496,19 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & pardouble nll2 = 0.0;const int num_batches = (n_ctx + n_batch - 1) / n_batch;+ const int n_seq = std::max(1, n_batch / n_ctx);++ GGML_ASSERT(n_batch < n_ctx || n_batch % n_ctx == 0);+ GGML_ASSERT(params.n_ctx == n_seq * n_ctx);++ llama_batch batch = llama_batch_init(std::min(n_batch, n_ctx*n_seq), 0, 1);std::vector<float> logits;if (num_batches > 1) {logits.reserve((size_t)n_ctx * n_vocab);}- fprintf(stderr, "%s: calculating perplexity over %d chunks, batch_size=%d\n", __func__, n_chunk, n_batch);+ fprintf(stderr, "%s: calculating perplexity over %d chunks, n_ctx=%d, batch_size=%d, n_seq=%d\n", __func__, n_chunk, n_ctx, n_batch, n_seq);std::vector<std::thread> workers(std::thread::hardware_concurrency() - 1);@@ -516,10 +521,26 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parlog_probs.resize(n_ctx * nv);}- for (int i = 0; i < n_chunk; ++i) {+ // We get the logits for all the tokens in the context window (params.n_ctx)+ // from llama_eval above. Now, based on https://huggingface.co/docs/transformers/perplexity,+ // calculate the perplexity over the last half of the window (so the model always has+ // some context to predict the token).+ //+ // We rely on the fact that attention in the forward pass only looks at previous+ // tokens here, so the logits returned for each token are an accurate representation+ // of what the model would have predicted at that point.+ //+ // Example, we have a context window of 512, we will compute perplexity for each of the+ // last 256 tokens. Then, we split the input up into context window size chunks to+ // process the entire prompt.+ const int first = n_ctx/2;++ for (int i = 0; i < n_chunk; i += n_seq) {const int start = i * n_ctx;const int end = start + n_ctx;+ const int n_seq_batch = std::min(n_seq, n_chunk - i);+const auto t_start = std::chrono::high_resolution_clock::now();// clear the KV cache@@ -529,34 +550,50 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parconst int batch_start = start + j * n_batch;const int batch_size = std::min(end - batch_start, n_batch);- // save original token and restore it after eval- const auto token_org = tokens[batch_start];+ batch.n_tokens = 0;+ for (int seq = 0; seq < n_seq_batch; seq++) {+ int seq_start = batch_start + seq*n_ctx;- // add BOS token for the first batch of each chunk- if (add_bos && j == 0) {- tokens[batch_start] = llama_token_bos(llama_get_model(ctx));+ // save original token and restore it after eval+ const auto token_org = tokens[seq_start];++ // add BOS token for the first batch of each chunk+ if (add_bos && j == 0) {+ tokens[seq_start] = llama_token_bos(llama_get_model(ctx));+ }++ for (int k = 0; k < batch_size; ++k) {+ const int idx = seq*n_ctx + k;+ batch.token[idx] = tokens[seq_start + k];+ batch.pos[idx] = j*n_batch + k;+ batch.n_seq_id[idx] = 1;+ batch.seq_id[idx][0] = seq;+ batch.logits[idx] = batch.pos[idx] >= first ? 1 : 0;+ }+ batch.n_tokens += batch_size;++ // restore the original token in case it was set to BOS+ tokens[seq_start] = token_org;}- if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {+ if (llama_decode(ctx, batch)) {fprintf(stderr, "%s : failed to eval\n", __func__);return {tokens, -1, logit_history, prob_history};}- // restore the original token in case it was set to BOS- tokens[batch_start] = token_org;-if (num_batches > 1) {const auto * batch_logits = llama_get_logits(ctx);logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab);}}- const auto t_end = std::chrono::high_resolution_clock::now();if (i == 0) {+ llama_synchronize(ctx);+ const auto t_end = std::chrono::high_resolution_clock::now();const float t_total = std::chrono::duration<float>(t_end - t_start).count();fprintf(stderr, "%s: %.2f seconds per pass - ETA ", __func__, t_total);- int total_seconds = (int)(t_total * n_chunk);+ int total_seconds = (int)(t_total*n_chunk/n_seq);if (total_seconds >= 60*60) {fprintf(stderr, "%d hours ", total_seconds / (60*60));total_seconds = total_seconds % (60*60);@@ -564,37 +601,31 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parfprintf(stderr, "%.2f minutes\n", total_seconds / 60.0);}- // We get the logits for all the tokens in the context window (params.n_ctx)- // from llama_eval above. Now, based on https://huggingface.co/docs/transformers/perplexity,- // calculate the perplexity over the last half of the window (so the model always has- // some context to predict the token).- //- // We rely on the fact that attention in the forward pass only looks at previous- // tokens here, so the logits returned for each token are an accurate representation- // of what the model would have predicted at that point.- //- // Example, we have a context window of 512, we will compute perplexity for each of the- // last 256 tokens. Then, we split the input up into context window size chunks to- // process the entire prompt.- const int first = n_ctx/2;- const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);- if (!params.logits_file.empty()) {- process_logits(logits_stream, n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,- workers, log_probs, nll, nll2);- } else {- process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,- workers, nll, nll2, logit_history.data() + start + first, prob_history.data() + start + first);- }- count += n_ctx - first - 1;-- // perplexity is e^(average negative log-likelihood)- if (params.ppl_output_type == 0) {- printf("[%d]%.4lf,", i + 1, std::exp(nll / count));- } else {- double av = nll/count;- double av2 = nll2/count - av*av;- if (av2 > 0) av2 = sqrt(av2/(count-1));- printf("%8d %.4lf %4lf %4lf\n", i*n_ctx, std::exp(nll / count), av, av2);+ for (int seq = 0; seq < n_seq_batch; seq++) {+ const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx);+ llama_token * tokens_data = tokens.data() + start + seq*n_ctx + first;+ if (!params.logits_file.empty()) {+ process_logits(logits_stream, n_vocab, all_logits + first*n_vocab,+ tokens_data, n_ctx - 1 - first,+ workers, log_probs, nll, nll2);+ } else {+ process_logits(n_vocab, all_logits + first*n_vocab,+ tokens_data, n_ctx - 1 - first,+ workers, nll, nll2,+ logit_history.data() + start + seq*n_ctx + first,+ prob_history.data() + start + seq*n_ctx + first);+ }+ count += n_ctx - first - 1;++ // perplexity is e^(average negative log-likelihood)+ if (params.ppl_output_type == 0) {+ printf("[%d]%.4lf,", i + seq + 1, std::exp(nll / count));+ } else {+ double av = nll/count;+ double av2 = nll2/count - av*av;+ if (av2 > 0) av2 = sqrt(av2/(count-1));+ printf("%8d %.4lf %4lf %4lf\n", i*n_ctx, std::exp(nll / count), av, av2);+ }}fflush(stdout);@@ -613,6 +644,8 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parprintf("Unexpected negative standard deviation of log(prob)\n");}+ llama_batch_free(batch);+return {tokens, ppl, logit_history, prob_history};}@@ -807,7 +840,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {const int n_batch = params.n_batch;const int max_tasks_per_batch = 32;- const int max_seq = 4*max_tasks_per_batch;+ const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_seq_max(ctx));llama_batch batch = llama_batch_init(n_ctx, 0, max_seq);@@ -1084,7 +1117,7 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) {const int n_batch = params.n_batch;const int max_tasks_per_batch = 128;- const int max_seq = 2*max_tasks_per_batch;+ const int max_seq = std::min(2*max_tasks_per_batch, (int) llama_n_seq_max(ctx));llama_batch batch = llama_batch_init(n_ctx, 0, max_seq);@@ -1436,7 +1469,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & paramsconst int n_batch = params.n_batch;const int max_tasks_per_batch = 32;- const int max_seq = 4*max_tasks_per_batch;+ const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_seq_max(ctx));llama_batch batch = llama_batch_init(n_ctx, 0, max_seq);@@ -1796,13 +1829,24 @@ int main(int argc, char ** argv) {gpt_params params;- params.n_batch = 512;if (!gpt_params_parse(argc, argv, params)) {return 1;}params.logits_all = true;- params.n_batch = std::min(params.n_batch, params.n_ctx);++ const int32_t n_ctx = params.n_ctx;++ const bool ppl = !params.hellaswag && !params.winogrande && !params.multiple_choice && !params.kl_divergence;+ if (ppl) {+ int n_seq = std::max(1, params.n_batch / n_ctx);+ int32_t n_kv = n_seq * n_ctx;+ params.n_parallel = n_seq;+ params.n_ctx = n_kv;+ params.n_batch = std::min(params.n_batch, n_kv);+ } else {+ params.n_batch = std::min(params.n_batch, params.n_ctx);+ }if (params.ppl_stride > 0) {fprintf(stderr, "Will perform strided perplexity calculation -> adjusting context size from %d to %d\n",@@ -1829,6 +1873,9 @@ int main(int argc, char ** argv) {llama_model * model;llama_context * ctx;+ // ensure there's at least enough seq_ids for HellaSwag+ params.n_parallel = std::max(4, params.n_parallel);+// load the model and apply lora adapter, if anystd::tie(model, ctx) = llama_init_from_gpt_params(params);if (model == NULL) {@@ -1858,7 +1905,7 @@ int main(int argc, char ** argv) {} else if (params.kl_divergence) {kl_divergence(ctx, params);} else {- results = perplexity(ctx, params);+ results = perplexity(ctx, params, n_ctx);}llama_print_timings(ctx);diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cppindex a8ae5131..7b606ca7 100644--- a/llama.cpp/sampling.cpp+++ b/llama.cpp/sampling.cpp@@ -20,6 +20,13 @@ struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_return nullptr;}+ // Ensure that there is a "root" node.+ if (result->parsed_grammar.symbol_ids.find("root") == result->parsed_grammar.symbol_ids.end()) {+ fprintf(stderr, "%s: grammar does not contain a 'root' symbol\n", __func__);+ delete result;+ return nullptr;+ }+std::vector<const llama_grammar_element *> grammar_rules(result->parsed_grammar.c_rules());result->grammar = llama_grammar_init(@@ -298,6 +305,77 @@ static llama_token llama_sampling_sample_impl(return id;}+static llama_token_data_array llama_sample_probability_distribution_impl(+ struct llama_sampling_context * ctx_sampling,+ struct llama_context * ctx_main,+ struct llama_context * ctx_cfg,+ const int idx) {+ const llama_sampling_params & params = ctx_sampling->params;++ const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));++ const int32_t penalty_last_n = params.penalty_last_n < 0 ? params.n_prev : params.penalty_last_n;+ const float penalty_repeat = params.penalty_repeat;+ const float penalty_freq = params.penalty_freq;+ const float penalty_present = params.penalty_present;+ const bool penalize_nl = params.penalize_nl;++ auto & prev = ctx_sampling->prev;+ auto & cur = ctx_sampling->cur;++ // Get a pointer to the logits+ float * logits = llama_get_logits_ith(ctx_main, idx);++ // Declare original_logits at the beginning of the function scope+ std::vector<float> original_logits;++ // apply params.logit_bias map+ for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {+ logits[it->first] += it->second;+ }++ if (ctx_cfg) {+ float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx);+ llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale);+ }++ cur.clear();++ for (llama_token token_id = 0; token_id < n_vocab; token_id++) {+ cur.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});+ }++ llama_token_data_array cur_p = { cur.data(), cur.size(), false };++ // apply penalties+ const auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev;+ const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n);+ if (penalty_tokens_used_size) {+ const float nl_logit = logits[llama_token_nl(llama_get_model(ctx_main))];++ llama_sample_repetition_penalties(ctx_main, &cur_p,+ penalty_tokens.data() + penalty_tokens.size() - penalty_tokens_used_size,+ penalty_tokens_used_size, penalty_repeat, penalty_freq, penalty_present);++ if (!penalize_nl) {+ for (size_t idx = 0; idx < cur_p.size; idx++) {+ if (cur_p.data[idx].id == llama_token_nl(llama_get_model(ctx_main))) {+ cur_p.data[idx].logit = nl_logit;+ break;+ }+ }+ }+ }++ // apply grammar checks+ if (ctx_sampling->grammar != NULL) {+ llama_sample_grammar(ctx_main, &cur_p, ctx_sampling->grammar);+ }++ llama_sample_softmax(ctx_main, &cur_p);+ return cur_p;+}+llama_token llama_sampling_sample(struct llama_sampling_context * ctx_sampling,struct llama_context * ctx_main,@@ -307,6 +385,14 @@ llama_token llama_sampling_sample(return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, false);}+llama_token_data_array llama_sampling_probability_distribution(+ struct llama_sampling_context * ctx_sampling,+ struct llama_context * ctx_main,+ struct llama_context * ctx_cfg,+ const int idx) {+ return llama_sample_probability_distribution_impl(ctx_sampling,ctx_main, ctx_cfg, idx);+}+void llama_sampling_accept(struct llama_sampling_context * ctx_sampling,struct llama_context * ctx_main,diff --git a/llama.cpp/sampling.h b/llama.cpp/sampling.hindex 95d87539..79a998be 100644--- a/llama.cpp/sampling.h+++ b/llama.cpp/sampling.h@@ -32,13 +32,13 @@ typedef struct llama_sampling_params {float dynatemp_range = 0.00f; // 0.0 = disabledfloat dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature samplerint32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)- float penalty_repeat = 1.10f; // 1.0 = disabled+ float penalty_repeat = 1.00f; // 1.0 = disabledfloat penalty_freq = 0.00f; // 0.0 = disabledfloat penalty_present = 0.00f; // 0.0 = disabledint32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0float mirostat_tau = 5.00f; // target entropyfloat mirostat_eta = 0.10f; // learning rate- bool penalize_nl = true; // consider newlines as a repeatable token+ bool penalize_nl = false; // consider newlines as a repeatable tokenstd::vector<llama_sampler_type> samplers_sequence = {llama_sampler_type::TOP_K,@@ -131,6 +131,13 @@ llama_token llama_sampling_sample(struct llama_context * ctx_cfg,int idx = 0);+// returns the probability that token of given id will be sampled+llama_token_data_array llama_sampling_probability_distribution(+ struct llama_sampling_context * ctx_sampling,+ struct llama_context * ctx_main,+ struct llama_context * ctx_cfg,+ int idx = 0);+void llama_sampling_accept(struct llama_sampling_context * ctx_sampling,struct llama_context * ctx_main,diff --git a/llama.cpp/server/httplib.h b/llama.cpp/server/httplib.hindex 28746000..f360bd93 100644--- a/llama.cpp/server/httplib.h+++ b/llama.cpp/server/httplib.h@@ -1,14 +1,14 @@//// httplib.h//-// Copyright (c) 2023 Yuji Hirose. All rights reserved.+// Copyright (c) 2024 Yuji Hirose. All rights reserved.// MIT License//#ifndef CPPHTTPLIB_HTTPLIB_H#define CPPHTTPLIB_HTTPLIB_H-#define CPPHTTPLIB_VERSION "0.12.2"+#define CPPHTTPLIB_VERSION "0.15.3"/** Configuration@@ -82,6 +82,10 @@#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192#endif+#ifndef CPPHTTPLIB_RANGE_MAX_COUNT+#define CPPHTTPLIB_RANGE_MAX_COUNT 1024+#endif+#ifndef CPPHTTPLIB_TCP_NODELAY#define CPPHTTPLIB_TCP_NODELAY false#endif@@ -160,10 +164,6 @@ using ssize_t = long;#define WSA_FLAG_NO_HANDLE_INHERIT 0x80#endif-#ifndef strcasecmp-#define strcasecmp _stricmp-#endif // strcasecmp-using socket_t = SOCKET;#ifdef CPPHTTPLIB_USE_POLL#define poll(fds, nfds, timeout) WSAPoll(fds, nfds, timeout)@@ -172,9 +172,15 @@ using socket_t = SOCKET;#else // not _WIN32#include <arpa/inet.h>-#ifndef _AIX+#if !defined(_AIX) && !defined(__MVS__)#include <ifaddrs.h>#endif+#ifdef __MVS__+#include <strings.h>+#ifndef NI_MAXHOST+#define NI_MAXHOST 1025+#endif+#endif#include <net/if.h>#include <netdb.h>#include <netinet/in.h>@@ -187,6 +193,7 @@ using socket_t = SOCKET;#endif#include <csignal>#include <pthread.h>+#include <sys/mman.h>#include <sys/select.h>#include <sys/socket.h>#include <sys/un.h>@@ -207,6 +214,7 @@ using socket_t = int;#include <condition_variable>#include <cstring>#include <errno.h>+#include <exception>#include <fcntl.h>#include <fstream>#include <functional>@@ -223,6 +231,9 @@ using socket_t = int;#include <string>#include <sys/stat.h>#include <thread>+#include <unordered_map>+#include <unordered_set>+#include <utility>#ifdef CPPHTTPLIB_OPENSSL_SUPPORT#ifdef _WIN32@@ -237,7 +248,6 @@ using socket_t = int;#ifdef _MSC_VER#pragma comment(lib, "crypt32.lib")-#pragma comment(lib, "cryptui.lib")#endif#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)#include <TargetConditionals.h>@@ -259,10 +269,8 @@ using socket_t = int;#include <iostream>#include <sstream>-#if OPENSSL_VERSION_NUMBER < 0x1010100fL-#error Sorry, OpenSSL versions prior to 1.1.1 are not supported-#elif OPENSSL_VERSION_NUMBER < 0x30000000L-#define SSL_get1_peer_certificate SSL_get_peer_certificate+#if OPENSSL_VERSION_NUMBER < 0x30000000L+#error Sorry, OpenSSL versions prior to 3.0.0 are not supported#endif#endif@@ -321,7 +329,7 @@ struct scope_exit {explicit scope_exit(std::function<void(void)> &&f): exit_function(std::move(f)), execute_on_destruction{true} {}- scope_exit(scope_exit &&rhs)+ scope_exit(scope_exit &&rhs) noexcept: exit_function(std::move(rhs.exit_function)),execute_on_destruction{rhs.execute_on_destruction} {rhs.release();@@ -344,6 +352,81 @@ private:} // namespace detail+enum StatusCode {+ // Information responses+ Continue_100 = 100,+ SwitchingProtocol_101 = 101,+ Processing_102 = 102,+ EarlyHints_103 = 103,++ // Successful responses+ OK_200 = 200,+ Created_201 = 201,+ Accepted_202 = 202,+ NonAuthoritativeInformation_203 = 203,+ NoContent_204 = 204,+ ResetContent_205 = 205,+ PartialContent_206 = 206,+ MultiStatus_207 = 207,+ AlreadyReported_208 = 208,+ IMUsed_226 = 226,++ // Redirection messages+ MultipleChoices_300 = 300,+ MovedPermanently_301 = 301,+ Found_302 = 302,+ SeeOther_303 = 303,+ NotModified_304 = 304,+ UseProxy_305 = 305,+ unused_306 = 306,+ TemporaryRedirect_307 = 307,+ PermanentRedirect_308 = 308,++ // Client error responses+ BadRequest_400 = 400,+ Unauthorized_401 = 401,+ PaymentRequired_402 = 402,+ Forbidden_403 = 403,+ NotFound_404 = 404,+ MethodNotAllowed_405 = 405,+ NotAcceptable_406 = 406,+ ProxyAuthenticationRequired_407 = 407,+ RequestTimeout_408 = 408,+ Conflict_409 = 409,+ Gone_410 = 410,+ LengthRequired_411 = 411,+ PreconditionFailed_412 = 412,+ PayloadTooLarge_413 = 413,+ UriTooLong_414 = 414,+ UnsupportedMediaType_415 = 415,+ RangeNotSatisfiable_416 = 416,+ ExpectationFailed_417 = 417,+ ImATeapot_418 = 418,+ MisdirectedRequest_421 = 421,+ UnprocessableContent_422 = 422,+ Locked_423 = 423,+ FailedDependency_424 = 424,+ TooEarly_425 = 425,+ UpgradeRequired_426 = 426,+ PreconditionRequired_428 = 428,+ TooManyRequests_429 = 429,+ RequestHeaderFieldsTooLarge_431 = 431,+ UnavailableForLegalReasons_451 = 451,++ // Server error responses+ InternalServerError_500 = 500,+ NotImplemented_501 = 501,+ BadGateway_502 = 502,+ ServiceUnavailable_503 = 503,+ GatewayTimeout_504 = 504,+ HttpVersionNotSupported_505 = 505,+ VariantAlsoNegotiates_506 = 506,+ InsufficientStorage_507 = 507,+ LoopDetected_508 = 508,+ NotExtended_510 = 510,+ NetworkAuthenticationRequired_511 = 511,+};+using Headers = std::multimap<std::string, std::string, detail::ci>;using Params = std::multimap<std::string, std::string>;@@ -373,17 +456,18 @@ public:DataSink &operator=(DataSink &&) = delete;std::function<bool(const char *data, size_t data_len)> write;+ std::function<bool()> is_writable;std::function<void()> done;std::function<void(const Headers &trailer)> done_with_trailer;std::ostream os;private:- class data_sink_streambuf : public std::streambuf {+ class data_sink_streambuf final : public std::streambuf {public:explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {}protected:- std::streamsize xsputn(const char *s, std::streamsize n) {+ std::streamsize xsputn(const char *s, std::streamsize n) override {sink_.write(s, static_cast<size_t>(n));return n;}@@ -465,6 +549,7 @@ struct Request {MultipartFormDataMap files;Ranges ranges;Match matches;+ std::unordered_map<std::string, std::string> path_params;// for clientResponseHandler response_handler;@@ -476,8 +561,7 @@ struct Request {bool has_header(const std::string &key) const;std::string get_header_value(const std::string &key, size_t id = 0) const;- template <typename T>- T get_header_value(const std::string &key, size_t id = 0) const;+ uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const;size_t get_header_value_count(const std::string &key) const;void set_header(const std::string &key, const std::string &val);@@ -509,14 +593,14 @@ struct Response {bool has_header(const std::string &key) const;std::string get_header_value(const std::string &key, size_t id = 0) const;- template <typename T>- T get_header_value(const std::string &key, size_t id = 0) const;+ uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const;size_t get_header_value_count(const std::string &key) const;void set_header(const std::string &key, const std::string &val);- void set_redirect(const std::string &url, int status = 302);+ void set_redirect(const std::string &url, int status = StatusCode::Found_302);void set_content(const char *s, size_t n, const std::string &content_type);void set_content(const std::string &s, const std::string &content_type);+ void set_content(std::string &&s, const std::string &content_type);void set_content_provider(size_t length, const std::string &content_type, ContentProvider provider,@@ -573,15 +657,16 @@ public:TaskQueue() = default;virtual ~TaskQueue() = default;- virtual void enqueue(std::function<void()> fn) = 0;+ virtual bool enqueue(std::function<void()> fn) = 0;virtual void shutdown() = 0;virtual void on_idle() {}};-class ThreadPool : public TaskQueue {+class ThreadPool final : public TaskQueue {public:- explicit ThreadPool(size_t n) : shutdown_(false) {+ explicit ThreadPool(size_t n, size_t mqr = 0)+ : shutdown_(false), max_queued_requests_(mqr) {while (n) {threads_.emplace_back(worker(*this));n--;@@ -591,13 +676,17 @@ public:ThreadPool(const ThreadPool &) = delete;~ThreadPool() override = default;- void enqueue(std::function<void()> fn) override {+ bool enqueue(std::function<void()> fn) override {{std::unique_lock<std::mutex> lock(mutex_);+ if (max_queued_requests_ > 0 && jobs_.size() >= max_queued_requests_) {+ return false;+ }jobs_.push_back(std::move(fn));}cond_.notify_one();+ return true;}void shutdown() override {@@ -647,6 +736,7 @@ private:std::list<std::function<void()>> jobs_;bool shutdown_;+ size_t max_queued_requests_ = 0;std::condition_variable cond_;std::mutex mutex_;@@ -658,6 +748,82 @@ using SocketOptions = std::function<void(socket_t sock)>;void default_socket_options(socket_t sock);+const char *status_message(int status);++std::string get_bearer_token_auth(const Request &req);++namespace detail {++class MatcherBase {+public:+ virtual ~MatcherBase() = default;++ // Match request path and populate its matches and+ virtual bool match(Request &request) const = 0;+};++/**+ * Captures parameters in request path and stores them in Request::path_params+ *+ * Capture name is a substring of a pattern from : to /.+ * The rest of the pattern is matched agains the request path directly+ * Parameters are captured starting from the next character after+ * the end of the last matched static pattern fragment until the next /.+ *+ * Example pattern:+ * "/path/fragments/:capture/more/fragments/:second_capture"+ * Static fragments:+ * "/path/fragments/", "more/fragments/"+ *+ * Given the following request path:+ * "/path/fragments/:1/more/fragments/:2"+ * the resulting capture will be+ * {{"capture", "1"}, {"second_capture", "2"}}+ */+class PathParamsMatcher final : public MatcherBase {+public:+ PathParamsMatcher(const std::string &pattern);++ bool match(Request &request) const override;++private:+ static constexpr char marker = ':';+ // Treat segment separators as the end of path parameter capture+ // Does not need to handle query parameters as they are parsed before path+ // matching+ static constexpr char separator = '/';++ // Contains static path fragments to match against, excluding the '/' after+ // path params+ // Fragments are separated by path params+ std::vector<std::string> static_fragments_;+ // Stores the names of the path parameters to be used as keys in the+ // Request::path_params map+ std::vector<std::string> param_names_;+};++/**+ * Performs std::regex_match on request path+ * and stores the result in Request::matches+ *+ * Note that regex match is performed directly on the whole request.+ * This means that wildcard patterns may match multiple path segments with /:+ * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end".+ */+class RegexMatcher final : public MatcherBase {+public:+ RegexMatcher(const std::string &pattern) : regex_(pattern) {}++ bool match(Request &request) const override;++private:+ std::regex regex_;+};++ssize_t write_headers(Stream &strm, const Headers &headers);++} // namespace detail+class Server {public:using Handler = std::function<void(const Request &, Response &)>;@@ -702,6 +868,7 @@ public:bool remove_mount_point(const std::string &mount_point);Server &set_file_extension_and_mimetype_mapping(const std::string &ext,const std::string &mime);+ Server &set_default_file_mimetype(const std::string &mime);Server &set_file_request_handler(Handler handler);Server &set_error_handler(HandlerWithResponse handler);@@ -718,6 +885,8 @@ public:Server &set_socket_options(SocketOptions socket_options);Server &set_default_headers(Headers headers);+ Server &+ set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);Server &set_keep_alive_max_count(size_t count);Server &set_keep_alive_timeout(time_t sec);@@ -765,9 +934,14 @@ protected:size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH;private:- using Handlers = std::vector<std::pair<std::regex, Handler>>;+ using Handlers =+ std::vector<std::pair<std::unique_ptr<detail::MatcherBase>, Handler>>;using HandlersForContentReader =- std::vector<std::pair<std::regex, HandlerWithContentReader>>;+ std::vector<std::pair<std::unique_ptr<detail::MatcherBase>,+ HandlerWithContentReader>>;++ static std::unique_ptr<detail::MatcherBase>+ make_matcher(const std::string &pattern);socket_t create_server_socket(const std::string &host, int port,int socket_flags,@@ -778,16 +952,16 @@ private:bool routing(Request &req, Response &res, Stream &strm);bool handle_file_request(const Request &req, Response &res,bool head = false);- bool dispatch_request(Request &req, Response &res, const Handlers &handlers);- bool- dispatch_request_for_content_reader(Request &req, Response &res,- ContentReader content_reader,- const HandlersForContentReader &handlers);+ bool dispatch_request(Request &req, Response &res,+ const Handlers &handlers) const;+ bool dispatch_request_for_content_reader(+ Request &req, Response &res, ContentReader content_reader,+ const HandlersForContentReader &handlers) const;- bool parse_request_line(const char *s, Request &req);+ bool parse_request_line(const char *s, Request &req) const;void apply_ranges(const Request &req, Response &res,- std::string &content_type, std::string &boundary);- bool write_response(Stream &strm, bool close_connection, const Request &req,+ std::string &content_type, std::string &boundary) const;+ bool write_response(Stream &strm, bool close_connection, Request &req,Response &res);bool write_response_with_content(Stream &strm, bool close_connection,const Request &req, Response &res);@@ -806,21 +980,23 @@ private:bool read_content_core(Stream &strm, Request &req, Response &res,ContentReceiver receiver,MultipartContentHeader multipart_header,- ContentReceiver multipart_receiver);+ ContentReceiver multipart_receiver) const;virtual bool process_and_close_socket(socket_t sock);+ std::atomic<bool> is_running_{false};+ std::atomic<bool> done_{false};+struct MountPointEntry {std::string mount_point;std::string base_dir;Headers headers;};std::vector<MountPointEntry> base_dirs_;-- std::atomic<bool> is_running_{false};- std::atomic<bool> done_{false};std::map<std::string, std::string> file_extension_and_mimetype_map_;+ std::string default_file_mimetype_ = "application/octet-stream";Handler file_request_handler_;+Handlers get_handlers_;Handlers post_handlers_;HandlersForContentReader post_handlers_for_content_reader_;@@ -831,18 +1007,22 @@ private:Handlers delete_handlers_;HandlersForContentReader delete_handlers_for_content_reader_;Handlers options_handlers_;+HandlerWithResponse error_handler_;ExceptionHandler exception_handler_;HandlerWithResponse pre_routing_handler_;Handler post_routing_handler_;- Logger logger_;Expect100ContinueHandler expect_100_continue_handler_;+ Logger logger_;+int address_family_ = AF_UNSPEC;bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY;SocketOptions socket_options_ = default_socket_options;Headers default_headers_;+ std::function<ssize_t(Stream &, Headers &)> header_writer_ =+ detail::write_headers;};enum class Error {@@ -860,17 +1040,19 @@ enum class Error {UnsupportedMultipartBoundaryChars,Compression,ConnectionTimeout,+ ProxyConnection,// For internal use onlySSLPeerCouldBeClosed_,};-std::string to_string(const Error error);+std::string to_string(Error error);std::ostream &operator<<(std::ostream &os, const Error &obj);class Result {public:+ Result() = default;Result(std::unique_ptr<Response> &&res, Error err,Headers &&request_headers = Headers{}): res_(std::move(res)), err_(err),@@ -893,13 +1075,13 @@ public:bool has_request_header(const std::string &key) const;std::string get_request_header_value(const std::string &key,size_t id = 0) const;- template <typename T>- T get_request_header_value(const std::string &key, size_t id = 0) const;+ uint64_t get_request_header_value_u64(const std::string &key,+ size_t id = 0) const;size_t get_request_header_value_count(const std::string &key) const;private:std::unique_ptr<Response> res_;- Error err_;+ Error err_ = Error::Unknown;Headers request_headers_;};@@ -1059,16 +1241,21 @@ public:bool send(Request &req, Response &res, Error &error);Result send(const Request &req);- size_t is_socket_open() const;+ void stop();- socket_t socket() const;+ std::string host() const;+ int port() const;- void stop();+ size_t is_socket_open() const;+ socket_t socket() const;void set_hostname_addr_map(std::map<std::string, std::string> addr_map);void set_default_headers(Headers headers);+ void+ set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);+void set_address_family(int family);void set_tcp_nodelay(bool on);void set_socket_options(SocketOptions socket_options);@@ -1117,6 +1304,7 @@ public:void set_ca_cert_path(const std::string &ca_cert_file_path,const std::string &ca_cert_dir_path = std::string());void set_ca_cert_store(X509_STORE *ca_cert_store);+ X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size) const;#endif#ifdef CPPHTTPLIB_OPENSSL_SUPPORT@@ -1145,14 +1333,14 @@ protected:// Also, shutdown_ssl and close_socket should also NOT be called concurrently// with a DIFFERENT thread sending requests using that socket.virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully);- void shutdown_socket(Socket &socket);+ void shutdown_socket(Socket &socket) const;void close_socket(Socket &socket);bool process_request(Stream &strm, Request &req, Response &res,bool close_connection, Error &error);bool write_content_with_provider(Stream &strm, const Request &req,- Error &error);+ Error &error) const;void copy_settings(const ClientImpl &rhs);@@ -1177,6 +1365,10 @@ protected:// Default headersHeaders default_headers_;+ // Header writer+ std::function<ssize_t(Stream &, Headers &)> header_writer_ =+ detail::write_headers;+// Settingsstd::string client_cert_path_;std::string client_key_path_;@@ -1239,7 +1431,8 @@ private:Result send_(Request &&req);socket_t create_client_socket(Error &error) const;- bool read_response_line(Stream &strm, const Request &req, Response &res);+ bool read_response_line(Stream &strm, const Request &req,+ Response &res) const;bool write_request(Stream &strm, Request &req, bool close_connection,Error &error);bool redirect(Request &req, Response &res, Error &error);@@ -1258,7 +1451,7 @@ private:const std::string &content_type);ContentProviderWithoutLength get_multipart_content_provider(const std::string &boundary, const MultipartFormDataItems &items,- const MultipartFormDataProviderItems &provider_items);+ const MultipartFormDataProviderItems &provider_items) const;std::string adjust_host_string(const std::string &host) const;@@ -1431,16 +1624,21 @@ public:bool send(Request &req, Response &res, Error &error);Result send(const Request &req);- size_t is_socket_open() const;+ void stop();- socket_t socket() const;+ std::string host() const;+ int port() const;- void stop();+ size_t is_socket_open() const;+ socket_t socket() const;void set_hostname_addr_map(std::map<std::string, std::string> addr_map);void set_default_headers(Headers headers);+ void+ set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);+void set_address_family(int family);void set_tcp_nodelay(bool on);void set_socket_options(SocketOptions socket_options);@@ -1497,6 +1695,7 @@ public:const std::string &ca_cert_dir_path = std::string());void set_ca_cert_store(X509_STORE *ca_cert_store);+ void load_ca_cert_store(const char *ca_cert, std::size_t size);long get_openssl_verify_result() const;@@ -1538,7 +1737,7 @@ private:std::mutex ctx_mutex_;};-class SSLClient : public ClientImpl {+class SSLClient final : public ClientImpl {public:explicit SSLClient(const std::string &host);@@ -1546,16 +1745,19 @@ public:explicit SSLClient(const std::string &host, int port,const std::string &client_cert_path,- const std::string &client_key_path);+ const std::string &client_key_path,+ const std::string &private_key_password = std::string());explicit SSLClient(const std::string &host, int port, X509 *client_cert,- EVP_PKEY *client_key);+ EVP_PKEY *client_key,+ const std::string &private_key_password = std::string());~SSLClient() override;bool is_valid() const override;void set_ca_cert_store(X509_STORE *ca_cert_store);+ void load_ca_cert_store(const char *ca_cert, std::size_t size);long get_openssl_verify_result() const;@@ -1564,7 +1766,7 @@ public:private:bool create_and_connect_socket(Socket &socket, Error &error) override;void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override;- void shutdown_ssl_impl(Socket &socket, bool shutdown_socket);+ void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully);bool process_socket(const Socket &socket,std::function<bool(Stream &strm)> callback) override;@@ -1608,15 +1810,9 @@ inline void duration_to_sec_and_usec(const T &duration, U callback) {callback(static_cast<time_t>(sec), static_cast<time_t>(usec));}-template <typename T>-inline T get_header_value(const Headers & /*headers*/,- const std::string & /*key*/, size_t /*id*/ = 0,- uint64_t /*def*/ = 0) {}--template <>-inline uint64_t get_header_value<uint64_t>(const Headers &headers,- const std::string &key, size_t id,- uint64_t def) {+inline uint64_t get_header_value_u64(const Headers &headers,+ const std::string &key, size_t id,+ uint64_t def) {auto rng = headers.equal_range(key);auto it = rng.first;std::advance(it, static_cast<ssize_t>(id));@@ -1628,14 +1824,14 @@ inline uint64_t get_header_value<uint64_t>(const Headers &headers,} // namespace detail-template <typename T>-inline T Request::get_header_value(const std::string &key, size_t id) const {- return detail::get_header_value<T>(headers, key, id, 0);+inline uint64_t Request::get_header_value_u64(const std::string &key,+ size_t id) const {+ return detail::get_header_value_u64(headers, key, id, 0);}-template <typename T>-inline T Response::get_header_value(const std::string &key, size_t id) const {- return detail::get_header_value<T>(headers, key, id, 0);+inline uint64_t Response::get_header_value_u64(const std::string &key,+ size_t id) const {+ return detail::get_header_value_u64(headers, key, id, 0);}template <typename... Args>@@ -1665,21 +1861,106 @@ inline ssize_t Stream::write_format(const char *fmt, const Args &...args) {inline void default_socket_options(socket_t sock) {int yes = 1;#ifdef _WIN32- setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char *>(&yes),- sizeof(yes));+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,+ reinterpret_cast<const char *>(&yes), sizeof(yes));setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,- reinterpret_cast<char *>(&yes), sizeof(yes));+ reinterpret_cast<const char *>(&yes), sizeof(yes));#else#ifdef SO_REUSEPORT- setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast<void *>(&yes),- sizeof(yes));+ setsockopt(sock, SOL_SOCKET, SO_REUSEPORT,+ reinterpret_cast<const void *>(&yes), sizeof(yes));#else- setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<void *>(&yes),- sizeof(yes));+ setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,+ reinterpret_cast<const void *>(&yes), sizeof(yes));#endif#endif}+inline const char *status_message(int status) {+ switch (status) {+ case StatusCode::Continue_100: return "Continue";+ case StatusCode::SwitchingProtocol_101: return "Switching Protocol";+ case StatusCode::Processing_102: return "Processing";+ case StatusCode::EarlyHints_103: return "Early Hints";+ case StatusCode::OK_200: return "OK";+ case StatusCode::Created_201: return "Created";+ case StatusCode::Accepted_202: return "Accepted";+ case StatusCode::NonAuthoritativeInformation_203:+ return "Non-Authoritative Information";+ case StatusCode::NoContent_204: return "No Content";+ case StatusCode::ResetContent_205: return "Reset Content";+ case StatusCode::PartialContent_206: return "Partial Content";+ case StatusCode::MultiStatus_207: return "Multi-Status";+ case StatusCode::AlreadyReported_208: return "Already Reported";+ case StatusCode::IMUsed_226: return "IM Used";+ case StatusCode::MultipleChoices_300: return "Multiple Choices";+ case StatusCode::MovedPermanently_301: return "Moved Permanently";+ case StatusCode::Found_302: return "Found";+ case StatusCode::SeeOther_303: return "See Other";+ case StatusCode::NotModified_304: return "Not Modified";+ case StatusCode::UseProxy_305: return "Use Proxy";+ case StatusCode::unused_306: return "unused";+ case StatusCode::TemporaryRedirect_307: return "Temporary Redirect";+ case StatusCode::PermanentRedirect_308: return "Permanent Redirect";+ case StatusCode::BadRequest_400: return "Bad Request";+ case StatusCode::Unauthorized_401: return "Unauthorized";+ case StatusCode::PaymentRequired_402: return "Payment Required";+ case StatusCode::Forbidden_403: return "Forbidden";+ case StatusCode::NotFound_404: return "Not Found";+ case StatusCode::MethodNotAllowed_405: return "Method Not Allowed";+ case StatusCode::NotAcceptable_406: return "Not Acceptable";+ case StatusCode::ProxyAuthenticationRequired_407:+ return "Proxy Authentication Required";+ case StatusCode::RequestTimeout_408: return "Request Timeout";+ case StatusCode::Conflict_409: return "Conflict";+ case StatusCode::Gone_410: return "Gone";+ case StatusCode::LengthRequired_411: return "Length Required";+ case StatusCode::PreconditionFailed_412: return "Precondition Failed";+ case StatusCode::PayloadTooLarge_413: return "Payload Too Large";+ case StatusCode::UriTooLong_414: return "URI Too Long";+ case StatusCode::UnsupportedMediaType_415: return "Unsupported Media Type";+ case StatusCode::RangeNotSatisfiable_416: return "Range Not Satisfiable";+ case StatusCode::ExpectationFailed_417: return "Expectation Failed";+ case StatusCode::ImATeapot_418: return "I'm a teapot";+ case StatusCode::MisdirectedRequest_421: return "Misdirected Request";+ case StatusCode::UnprocessableContent_422: return "Unprocessable Content";+ case StatusCode::Locked_423: return "Locked";+ case StatusCode::FailedDependency_424: return "Failed Dependency";+ case StatusCode::TooEarly_425: return "Too Early";+ case StatusCode::UpgradeRequired_426: return "Upgrade Required";+ case StatusCode::PreconditionRequired_428: return "Precondition Required";+ case StatusCode::TooManyRequests_429: return "Too Many Requests";+ case StatusCode::RequestHeaderFieldsTooLarge_431:+ return "Request Header Fields Too Large";+ case StatusCode::UnavailableForLegalReasons_451:+ return "Unavailable For Legal Reasons";+ case StatusCode::NotImplemented_501: return "Not Implemented";+ case StatusCode::BadGateway_502: return "Bad Gateway";+ case StatusCode::ServiceUnavailable_503: return "Service Unavailable";+ case StatusCode::GatewayTimeout_504: return "Gateway Timeout";+ case StatusCode::HttpVersionNotSupported_505:+ return "HTTP Version Not Supported";+ case StatusCode::VariantAlsoNegotiates_506: return "Variant Also Negotiates";+ case StatusCode::InsufficientStorage_507: return "Insufficient Storage";+ case StatusCode::LoopDetected_508: return "Loop Detected";+ case StatusCode::NotExtended_510: return "Not Extended";+ case StatusCode::NetworkAuthenticationRequired_511:+ return "Network Authentication Required";++ default:+ case StatusCode::InternalServerError_500: return "Internal Server Error";+ }+}++inline std::string get_bearer_token_auth(const Request &req) {+ if (req.has_header("Authorization")) {+ static std::string BearerHeaderPrefix = "Bearer ";+ return req.get_header_value("Authorization")+ .substr(BearerHeaderPrefix.length());+ }+ return "";+}+template <class Rep, class Period>inline Server &Server::set_read_timeout(const std::chrono::duration<Rep, Period> &duration) {@@ -1720,6 +2001,7 @@ inline std::string to_string(const Error error) {return "Unsupported HTTP multipart boundary characters";case Error::Compression: return "Compression failed";case Error::ConnectionTimeout: return "Connection timed out";+ case Error::ProxyConnection: return "Proxy connection failed";case Error::Unknown: return "Unknown";default: break;}@@ -1733,10 +2015,9 @@ inline std::ostream &operator<<(std::ostream &os, const Error &obj) {return os;}-template <typename T>-inline T Result::get_request_header_value(const std::string &key,- size_t id) const {- return detail::get_header_value<T>(request_headers_, key, id, 0);+inline uint64_t Result::get_request_header_value_u64(const std::string &key,+ size_t id) const {+ return detail::get_header_value_u64(request_headers_, key, id, 0);}template <class Rep, class Period>@@ -1790,7 +2071,7 @@ void hosted_at(const std::string &hostname, std::vector<std::string> &addrs);std::string append_query_params(const std::string &path, const Params ¶ms);-std::pair<std::string, std::string> make_range_header(Ranges ranges);+std::pair<std::string, std::string> make_range_header(const Ranges &ranges);std::pair<std::string, std::string>make_basic_authentication_header(const std::string &username,@@ -1810,6 +2091,9 @@ std::string trim_copy(const std::string &s);void split(const char *b, const char *e, char d,std::function<void(const char *, const char *)> fn);+void split(const char *b, const char *e, char d, size_t m,+ std::function<void(const char *, const char *)> fn);+bool process_client_socket(socket_t sock, time_t read_timeout_sec,time_t read_timeout_usec, time_t write_timeout_sec,time_t write_timeout_usec,@@ -1844,7 +2128,7 @@ enum class EncodingType { None = 0, Gzip, Brotli };EncodingType encoding_type(const Request &req, const Response &res);-class BufferStream : public Stream {+class BufferStream final : public Stream {public:BufferStream() = default;~BufferStream() override = default;@@ -1884,19 +2168,19 @@ public:Callback callback) = 0;};-class nocompressor : public compressor {+class nocompressor final : public compressor {public:- virtual ~nocompressor() = default;+ ~nocompressor() override = default;bool compress(const char *data, size_t data_length, bool /*last*/,Callback callback) override;};#ifdef CPPHTTPLIB_ZLIB_SUPPORT-class gzip_compressor : public compressor {+class gzip_compressor final : public compressor {public:gzip_compressor();- ~gzip_compressor();+ ~gzip_compressor() override;bool compress(const char *data, size_t data_length, bool last,Callback callback) override;@@ -1906,10 +2190,10 @@ private:z_stream strm_;};-class gzip_decompressor : public decompressor {+class gzip_decompressor final : public decompressor {public:gzip_decompressor();- ~gzip_decompressor();+ ~gzip_decompressor() override;bool is_valid() const override;@@ -1923,7 +2207,7 @@ private:#endif#ifdef CPPHTTPLIB_BROTLI_SUPPORT-class brotli_compressor : public compressor {+class brotli_compressor final : public compressor {public:brotli_compressor();~brotli_compressor();@@ -1935,7 +2219,7 @@ private:BrotliEncoderState *state_ = nullptr;};-class brotli_decompressor : public decompressor {+class brotli_decompressor final : public decompressor {public:brotli_decompressor();~brotli_decompressor();@@ -1972,6 +2256,29 @@ private:std::string glowable_buffer_;};+class mmap {+public:+ mmap(const char *path);+ ~mmap();++ bool open(const char *path);+ void close();++ bool is_open() const;+ size_t size() const;+ const char *data() const;++private:+#if defined(_WIN32)+ HANDLE hFile_;+ HANDLE hMapping_;+#else+ int fd_;+#endif+ size_t size_;+ void *addr_;+};+} // namespace detail// ----------------------------------------------------------------------------@@ -2003,7 +2310,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,val = 0;for (; cnt; i++, cnt--) {if (!s[i]) { return false; }- int v = 0;+ auto v = 0;if (is_hex(s[i], v)) {val = val * 16 + v;} else {@@ -2014,7 +2321,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,}inline std::string from_i_to_hex(size_t n) {- const char *charset = "0123456789abcdef";+ static const auto charset = "0123456789abcdef";std::string ret;do {ret = charset[n & 15] + ret;@@ -2025,7 +2332,7 @@ inline std::string from_i_to_hex(size_t n) {inline size_t to_utf8(int code, char *buff) {if (code < 0x0080) {- buff[0] = (code & 0x7F);+ buff[0] = static_cast<char>(code & 0x7F);return 1;} else if (code < 0x0800) {buff[0] = static_cast<char>(0xC0 | ((code >> 6) & 0x1F));@@ -2064,8 +2371,8 @@ inline std::string base64_encode(const std::string &in) {std::string out;out.reserve(in.size());- int val = 0;- int valb = -6;+ auto val = 0;+ auto valb = -6;for (auto c : in) {val = (val << 8) + static_cast<uint8_t>(c);@@ -2112,6 +2419,11 @@ inline bool is_valid_path(const std::string &path) {// Read componentauto beg = i;while (i < path.size() && path[i] != '/') {+ if (path[i] == '\0') {+ return false;+ } else if (path[i] == '\\') {+ return false;+ }i++;}@@ -2196,7 +2508,7 @@ inline std::string decode_url(const std::string &s,for (size_t i = 0; i < s.size(); i++) {if (s[i] == '%' && i + 1 < s.size()) {if (s[i + 1] == 'u') {- int val = 0;+ auto val = 0;if (from_hex_to_i(s, i + 2, 4, val)) {// 4 digits Unicode codeschar buff[4];@@ -2207,7 +2519,7 @@ inline std::string decode_url(const std::string &s,result += s[i];}} else {- int val = 0;+ auto val = 0;if (from_hex_to_i(s, i + 1, 2, val)) {// 2 digits hex codesresult += static_cast<char>(val);@@ -2260,16 +2572,30 @@ inline std::string trim_copy(const std::string &s) {return s.substr(r.first, r.second - r.first);}+inline std::string trim_double_quotes_copy(const std::string &s) {+ if (s.length() >= 2 && s.front() == '"' && s.back() == '"') {+ return s.substr(1, s.size() - 2);+ }+ return s;+}+inline void split(const char *b, const char *e, char d,std::function<void(const char *, const char *)> fn) {+ return split(b, e, d, (std::numeric_limits<size_t>::max)(), std::move(fn));+}++inline void split(const char *b, const char *e, char d, size_t m,+ std::function<void(const char *, const char *)> fn) {size_t i = 0;size_t beg = 0;+ size_t count = 1;while (e ? (b + i < e) : (b[i] != '\0')) {- if (b[i] == d) {+ if (b[i] == d && count < m) {auto r = trim(b, e, beg, i);if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }beg = i + 1;+ count++;}i++;}@@ -2345,6 +2671,105 @@ inline void stream_line_reader::append(char c) {}}+inline mmap::mmap(const char *path)+#if defined(_WIN32)+ : hFile_(NULL), hMapping_(NULL)+#else+ : fd_(-1)+#endif+ ,+ size_(0), addr_(nullptr) {+ open(path);+}++inline mmap::~mmap() { close(); }++inline bool mmap::open(const char *path) {+ close();++#if defined(_WIN32)+ std::wstring wpath;+ for (size_t i = 0; i < strlen(path); i++) {+ wpath += path[i];+ }++ hFile_ = ::CreateFile2(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ,+ OPEN_EXISTING, NULL);++ if (hFile_ == INVALID_HANDLE_VALUE) { return false; }++ LARGE_INTEGER size{};+ if (!::GetFileSizeEx(hFile_, &size)) { return false; }+ size_ = static_cast<size_t>(size.QuadPart);++ hMapping_ =+ ::CreateFileMappingFromApp(hFile_, NULL, PAGE_READONLY, size_, NULL);++ if (hMapping_ == NULL) {+ close();+ return false;+ }++ addr_ = ::MapViewOfFileFromApp(hMapping_, FILE_MAP_READ, 0, 0);+#else+ fd_ = ::open(path, O_RDONLY);+ if (fd_ == -1) { return false; }++ struct stat sb;+ if (fstat(fd_, &sb) == -1) {+ close();+ return false;+ }+ size_ = static_cast<size_t>(sb.st_size);++ addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0);+#endif++ if (addr_ == nullptr) {+ close();+ return false;+ }++ return true;+}++inline bool mmap::is_open() const { return addr_ != nullptr; }++inline size_t mmap::size() const { return size_; }++inline const char *mmap::data() const {+ return static_cast<const char *>(addr_);+}++inline void mmap::close() {+#if defined(_WIN32)+ if (addr_) {+ ::UnmapViewOfFile(addr_);+ addr_ = nullptr;+ }++ if (hMapping_) {+ ::CloseHandle(hMapping_);+ hMapping_ = NULL;+ }++ if (hFile_ != INVALID_HANDLE_VALUE) {+ ::CloseHandle(hFile_);+ hFile_ = INVALID_HANDLE_VALUE;+ }+#else+ if (addr_ != nullptr) {+ munmap(addr_, size_);+ addr_ = nullptr;+ }++ if (fd_ != -1) {+ ::close(fd_);+ fd_ = -1;+ }+#endif+ size_ = 0;+}inline int close_socket(socket_t sock) {#ifdef _WIN32return closesocket(sock);@@ -2354,7 +2779,7 @@ inline int close_socket(socket_t sock) {}template <typename T> inline ssize_t handle_EINTR(T fn) {- ssize_t res = false;+ ssize_t res = 0;while (true) {res = fn();if (res < 0 && errno == EINTR) { continue; }@@ -2399,7 +2824,7 @@ inline ssize_t select_read(socket_t sock, time_t sec, time_t usec) {return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });#else#ifndef _WIN32- if (sock >= FD_SETSIZE) { return 1; }+ if (sock >= FD_SETSIZE) { return -1; }#endiffd_set fds;@@ -2427,7 +2852,7 @@ inline ssize_t select_write(socket_t sock, time_t sec, time_t usec) {return handle_EINTR([&]() { return poll(&pfd_read, 1, timeout); });#else#ifndef _WIN32- if (sock >= FD_SETSIZE) { return 1; }+ if (sock >= FD_SETSIZE) { return -1; }#endiffd_set fds;@@ -2458,7 +2883,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec,if (poll_res == 0) { return Error::ConnectionTimeout; }if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) {- int error = 0;+ auto error = 0;socklen_t len = sizeof(error);auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,reinterpret_cast<char *>(&error), &len);@@ -2490,7 +2915,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec,if (ret == 0) { return Error::ConnectionTimeout; }if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) {- int error = 0;+ auto error = 0;socklen_t len = sizeof(error);auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,reinterpret_cast<char *>(&error), &len);@@ -2512,7 +2937,7 @@ inline bool is_socket_alive(socket_t sock) {return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0;}-class SocketStream : public Stream {+class SocketStream final : public Stream {public:SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,time_t write_timeout_sec, time_t write_timeout_usec);@@ -2537,11 +2962,11 @@ private:size_t read_buff_off_ = 0;size_t read_buff_content_size_ = 0;- static const size_t read_buff_size_ = 1024 * 4;+ static const size_t read_buff_size_ = 1024l * 4;};#ifdef CPPHTTPLIB_OPENSSL_SUPPORT-class SSLSocketStream : public Stream {+class SSLSocketStream final : public Stream {public:SSLSocketStream(socket_t sock, SSL *ssl, time_t read_timeout_sec,time_t read_timeout_usec, time_t write_timeout_sec,@@ -2666,7 +3091,7 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,#ifndef _WIN32if (hints.ai_family == AF_UNIX) {const auto addrlen = host.length();- if (addrlen > sizeof(sockaddr_un::sun_path)) return INVALID_SOCKET;+ if (addrlen > sizeof(sockaddr_un::sun_path)) { return INVALID_SOCKET; }auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol);if (sock != INVALID_SOCKET) {@@ -2735,17 +3160,27 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,#endifif (tcp_nodelay) {- int yes = 1;- setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char *>(&yes),- sizeof(yes));+ auto yes = 1;+#ifdef _WIN32+ setsockopt(sock, IPPROTO_TCP, TCP_NODELAY,+ reinterpret_cast<const char *>(&yes), sizeof(yes));+#else+ setsockopt(sock, IPPROTO_TCP, TCP_NODELAY,+ reinterpret_cast<const void *>(&yes), sizeof(yes));+#endif}if (socket_options) { socket_options(sock); }if (rp->ai_family == AF_INET6) {- int no = 0;- setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast<char *>(&no),- sizeof(no));+ auto no = 0;+#ifdef _WIN32+ setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY,+ reinterpret_cast<const char *>(&no), sizeof(no));+#else+ setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY,+ reinterpret_cast<const void *>(&no), sizeof(no));+#endif}// bind or connect@@ -2804,7 +3239,7 @@ inline bool bind_ip_address(socket_t sock, const std::string &host) {return ret;}-#if !defined _WIN32 && !defined ANDROID && !defined _AIX+#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__#define USE_IF2IP#endif@@ -2860,7 +3295,7 @@ inline socket_t create_client_socket(#ifdef USE_IF2IPauto ip_from_if = if2ip(address_family, intf);if (ip_from_if.empty()) { ip_from_if = intf; }- if (!bind_ip_address(sock2, ip_from_if.c_str())) {+ if (!bind_ip_address(sock2, ip_from_if)) {error = Error::BindIPAddress;return false;}@@ -2888,13 +3323,14 @@ inline socket_t create_client_socket(#ifdef _WIN32auto timeout = static_cast<uint32_t>(read_timeout_sec * 1000 +read_timeout_usec / 1000);- setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,- sizeof(timeout));+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO,+ reinterpret_cast<const char *>(&timeout), sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(read_timeout_sec);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec);- setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));+ setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO,+ reinterpret_cast<const void *>(&tv), sizeof(tv));#endif}{@@ -2902,13 +3338,14 @@ inline socket_t create_client_socket(#ifdef _WIN32auto timeout = static_cast<uint32_t>(write_timeout_sec * 1000 +write_timeout_usec / 1000);- setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,- sizeof(timeout));+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO,+ reinterpret_cast<const char *>(&timeout), sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(write_timeout_sec);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec);- setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));+ setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO,+ reinterpret_cast<const void *>(&tv), sizeof(tv));#endif}@@ -3008,18 +3445,20 @@ inline constexpr unsigned int operator"" _t(const char *s, size_t l) {} // namespace udl-inline const char *+inline std::stringfind_content_type(const std::string &path,- const std::map<std::string, std::string> &user_data) {+ const std::map<std::string, std::string> &user_data,+ const std::string &default_content_type) {auto ext = file_extension(path);auto it = user_data.find(ext);- if (it != user_data.end()) { return it->second.c_str(); }+ if (it != user_data.end()) { return it->second; }using udl::operator""_t;switch (str2tag(ext)) {- default: return nullptr;+ default: return default_content_type;+case "css"_t: return "text/css";case "csv"_t: return "text/csv";case "htm"_t:@@ -3072,76 +3511,6 @@ find_content_type(const std::string &path,}}-inline const char *status_message(int status) {- switch (status) {- case 100: return "Continue";- case 101: return "Switching Protocol";- case 102: return "Processing";- case 103: return "Early Hints";- case 200: return "OK";- case 201: return "Created";- case 202: return "Accepted";- case 203: return "Non-Authoritative Information";- case 204: return "No Content";- case 205: return "Reset Content";- case 206: return "Partial Content";- case 207: return "Multi-Status";- case 208: return "Already Reported";- case 226: return "IM Used";- case 300: return "Multiple Choice";- case 301: return "Moved Permanently";- case 302: return "Found";- case 303: return "See Other";- case 304: return "Not Modified";- case 305: return "Use Proxy";- case 306: return "unused";- case 307: return "Temporary Redirect";- case 308: return "Permanent Redirect";- case 400: return "Bad Request";- case 401: return "Unauthorized";- case 402: return "Payment Required";- case 403: return "Forbidden";- case 404: return "Not Found";- case 405: return "Method Not Allowed";- case 406: return "Not Acceptable";- case 407: return "Proxy Authentication Required";- case 408: return "Request Timeout";- case 409: return "Conflict";- case 410: return "Gone";- case 411: return "Length Required";- case 412: return "Precondition Failed";- case 413: return "Payload Too Large";- case 414: return "URI Too Long";- case 415: return "Unsupported Media Type";- case 416: return "Range Not Satisfiable";- case 417: return "Expectation Failed";- case 418: return "I'm a teapot";- case 421: return "Misdirected Request";- case 422: return "Unprocessable Entity";- case 423: return "Locked";- case 424: return "Failed Dependency";- case 425: return "Too Early";- case 426: return "Upgrade Required";- case 428: return "Precondition Required";- case 429: return "Too Many Requests";- case 431: return "Request Header Fields Too Large";- case 451: return "Unavailable For Legal Reasons";- case 501: return "Not Implemented";- case 502: return "Bad Gateway";- case 503: return "Service Unavailable";- case 504: return "Gateway Timeout";- case 505: return "HTTP Version Not Supported";- case 506: return "Variant Also Negotiates";- case 507: return "Insufficient Storage";- case 508: return "Loop Detected";- case 510: return "Not Extended";- case 511: return "Network Authentication Required";-- default:- case 500: return "Internal Server Error";- }-}-inline bool can_compress_content_type(const std::string &content_type) {using udl::operator""_t;@@ -3218,7 +3587,7 @@ inline bool gzip_compressor::compress(const char *data, size_t data_length,data += strm_.avail_in;auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH;- int ret = Z_OK;+ auto ret = Z_OK;std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};do {@@ -3262,7 +3631,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,Callback callback) {assert(is_valid_);- int ret = Z_OK;+ auto ret = Z_OK;do {constexpr size_t max_avail_in =@@ -3276,16 +3645,12 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,data += strm_.avail_in;std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};- while (strm_.avail_in > 0) {+ while (strm_.avail_in > 0 && ret == Z_OK) {strm_.avail_out = static_cast<uInt>(buff.size());strm_.next_out = reinterpret_cast<Bytef *>(buff.data());- auto prev_avail_in = strm_.avail_in;-ret = inflate(&strm_, Z_NO_FLUSH);- if (prev_avail_in - strm_.avail_in == 0) { return false; }-assert(ret != Z_STREAM_ERROR);switch (ret) {case Z_NEED_DICT:@@ -3298,7 +3663,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,}}- if (ret != Z_OK && ret != Z_STREAM_END) return false;+ if (ret != Z_OK && ret != Z_STREAM_END) { return false; }} while (data_length > 0);@@ -3367,7 +3732,7 @@ inline bool brotli_decompressor::decompress(const char *data,return 0;}- const uint8_t *next_in = (const uint8_t *)data;+ auto next_in = reinterpret_cast<const uint8_t *>(data);size_t avail_in = data_length;size_t total_out;@@ -3437,6 +3802,9 @@ inline bool parse_header(const char *beg, const char *end, T fn) {}if (p < end) {+ auto key_len = key_end - beg;+ if (!key_len) { return false; }+auto key = std::string(beg, key_end);auto val = compare_case_ignore(key, "Location")? std::string(p, end)@@ -3526,11 +3894,7 @@ inline bool read_content_without_length(Stream &strm,uint64_t r = 0;for (;;) {auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ);- if (n < 0) {- return false;- } else if (n == 0) {- return true;- }+ if (n <= 0) { return true; }if (!out(buf, static_cast<size_t>(n), r, 0)) { return false; }r += static_cast<uint64_t>(n);@@ -3566,7 +3930,7 @@ inline bool read_content_chunked(Stream &strm, T &x,if (!line_reader.getline()) { return false; }- if (strcmp(line_reader.ptr(), "\r\n")) { return false; }+ if (strcmp(line_reader.ptr(), "\r\n") != 0) { return false; }if (!line_reader.getline()) { return false; }}@@ -3576,7 +3940,7 @@ inline bool read_content_chunked(Stream &strm, T &x,// Trailerif (!line_reader.getline()) { return false; }- while (strcmp(line_reader.ptr(), "\r\n")) {+ while (strcmp(line_reader.ptr(), "\r\n") != 0) {if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }// Exclude line terminator@@ -3595,8 +3959,8 @@ inline bool read_content_chunked(Stream &strm, T &x,}inline bool is_chunked_transfer_encoding(const Headers &headers) {- return !strcasecmp(get_header_value(headers, "Transfer-Encoding", 0, ""),- "chunked");+ return compare_case_ignore(+ get_header_value(headers, "Transfer-Encoding", 0, ""), "chunked");}template <typename T, typename U>@@ -3611,14 +3975,14 @@ bool prepare_content_receiver(T &x, int &status,#ifdef CPPHTTPLIB_ZLIB_SUPPORTdecompressor = detail::make_unique<gzip_decompressor>();#else- status = 415;+ status = StatusCode::UnsupportedMediaType_415;return false;#endif} else if (encoding.find("br") != std::string::npos) {#ifdef CPPHTTPLIB_BROTLI_SUPPORTdecompressor = detail::make_unique<brotli_decompressor>();#else- status = 415;+ status = StatusCode::UnsupportedMediaType_415;return false;#endif}@@ -3634,7 +3998,7 @@ bool prepare_content_receiver(T &x, int &status,};return callback(std::move(out));} else {- status = 500;+ status = StatusCode::InternalServerError_500;return false;}}@@ -3662,7 +4026,7 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,} else if (!has_header(x.headers, "Content-Length")) {ret = read_content_without_length(strm, out);} else {- auto len = get_header_value<uint64_t>(x.headers, "Content-Length");+ auto len = get_header_value_u64(x.headers, "Content-Length", 0, 0);if (len > payload_max_length) {exceed_payload_max_length = true;skip_content_with_length(strm, len);@@ -3672,7 +4036,10 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,}}- if (!ret) { status = exceed_payload_max_length ? 413 : 400; }+ if (!ret) {+ status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413+ : StatusCode::BadRequest_400;+ }return ret;});} // namespace detail@@ -3720,6 +4087,8 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider,return ok;};+ data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };+while (offset < end_offset && !is_shutting_down()) {if (!strm.is_writable()) {error = Error::Write;@@ -3764,6 +4133,8 @@ write_content_without_length(Stream &strm,return ok;};+ data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };+data_sink.done = [&](void) { data_available = false; };while (data_available && !is_shutting_down()) {@@ -3814,6 +4185,8 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider,return ok;};+ data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };+auto done_with_trailer = [&](const Headers *trailer) {if (!ok) { return; }@@ -3898,7 +4271,8 @@ inline bool redirect(T &cli, Request &req, Response &res,new_req.path = path;new_req.redirect_count_ -= 1;- if (res.status == 303 && (req.method != "GET" && req.method != "HEAD")) {+ if (res.status == StatusCode::SeeOther_303 &&+ (req.method != "GET" && req.method != "HEAD")) {new_req.method = "GET";new_req.body.clear();new_req.headers.clear();@@ -3910,7 +4284,8 @@ inline bool redirect(T &cli, Request &req, Response &res,if (ret) {req = new_req;res = new_res;- res.location = location;++ if (res.location.empty()) { res.location = location; }}return ret;}@@ -3957,14 +4332,34 @@ inline bool parse_multipart_boundary(const std::string &content_type,if (pos == std::string::npos) { return false; }auto end = content_type.find(';', pos);auto beg = pos + strlen(boundary_keyword);- boundary = content_type.substr(beg, end - beg);- if (boundary.length() >= 2 && boundary.front() == '"' &&- boundary.back() == '"') {- boundary = boundary.substr(1, boundary.size() - 2);- }+ boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg));return !boundary.empty();}+inline void parse_disposition_params(const std::string &s, Params ¶ms) {+ std::set<std::string> cache;+ split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) {+ std::string kv(b, e);+ if (cache.find(kv) != cache.end()) { return; }+ cache.insert(kv);++ std::string key;+ std::string val;+ split(b, e, '=', [&](const char *b2, const char *e2) {+ if (key.empty()) {+ key.assign(b2, e2);+ } else {+ val.assign(b2, e2);+ }+ });++ if (!key.empty()) {+ params.emplace(trim_double_quotes_copy((key)),+ trim_double_quotes_copy((val)));+ }+ });+}+#ifdef CPPHTTPLIB_NO_EXCEPTIONSinline bool parse_range_header(const std::string &s, Ranges &ranges) {#else@@ -3975,9 +4370,9 @@ inline bool parse_range_header(const std::string &s, Ranges &ranges) try {if (std::regex_match(s, m, re_first_range)) {auto pos = static_cast<size_t>(m.position(1));auto len = static_cast<size_t>(m.length(1));- bool all_valid_ranges = true;+ auto all_valid_ranges = true;split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {- if (!all_valid_ranges) return;+ if (!all_valid_ranges) { return; }static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))");std::cmatch cm;if (std::regex_match(b, e, cm, re_another_range)) {@@ -4022,11 +4417,6 @@ public:bool parse(const char *buf, size_t n, const ContentReceiver &content_callback,const MultipartContentHeader &header_callback) {- // TODO: support 'filename*'- static const std::regex re_content_disposition(- R"~(^Content-Disposition:\s*form-data;\s*name="(.*?)"(?:;\s*filename="(.*?)")?(?:;\s*filename\*=\S+)?\s*$)~",- std::regex_constants::icase);-buf_append(buf, n);while (buf_size() > 0) {@@ -4059,18 +4449,54 @@ public:break;}- static const std::string header_name = "content-type:";const auto header = buf_head(pos);- if (start_with_case_ignore(header, header_name)) {- file_.content_type = trim_copy(header.substr(header_name.size()));++ if (!parse_header(header.data(), header.data() + header.size(),+ [&](std::string &&, std::string &&) {})) {+ is_valid_ = false;+ return false;+ }++ static const std::string header_content_type = "Content-Type:";++ if (start_with_case_ignore(header, header_content_type)) {+ file_.content_type =+ trim_copy(header.substr(header_content_type.size()));} else {+ static const std::regex re_content_disposition(+ R"~(^Content-Disposition:\s*form-data;\s*(.*)$)~",+ std::regex_constants::icase);+std::smatch m;if (std::regex_match(header, m, re_content_disposition)) {- file_.name = m[1];- file_.filename = m[2];- } else {- is_valid_ = false;- return false;+ Params params;+ parse_disposition_params(m[1], params);++ auto it = params.find("name");+ if (it != params.end()) {+ file_.name = it->second;+ } else {+ is_valid_ = false;+ return false;+ }++ it = params.find("filename");+ if (it != params.end()) { file_.filename = it->second; }++ it = params.find("filename*");+ if (it != params.end()) {+ // Only allow UTF-8 enconnding...+ static const std::regex re_rfc5987_encoding(+ R"~(^UTF-8''(.+?)$)~", std::regex_constants::icase);++ std::smatch m2;+ if (std::regex_match(it->second, m2, re_rfc5987_encoding)) {+ file_.filename = decode_url(m2[1], false); // override...+ } else {+ is_valid_ = false;+ return false;+ }+ }}}buf_erase(pos + crlf_.size());@@ -4108,9 +4534,9 @@ public:buf_erase(crlf_.size());state_ = 1;} else {- if (dash_crlf_.size() > buf_size()) { return true; }- if (buf_start_with(dash_crlf_)) {- buf_erase(dash_crlf_.size());+ if (dash_.size() > buf_size()) { return true; }+ if (buf_start_with(dash_)) {+ buf_erase(dash_.size());is_valid_ = true;buf_erase(buf_size()); // Remove epilogue} else {@@ -4143,7 +4569,6 @@ private:const std::string dash_ = "--";const std::string crlf_ = "\r\n";- const std::string dash_crlf_ = "--\r\n";std::string boundary_;std::string dash_boundary_crlf_;std::string crlf_dash_boundary_;@@ -4230,28 +4655,32 @@ inline std::string to_lower(const char *beg, const char *end) {return out;}-inline std::string make_multipart_data_boundary() {+inline std::string random_string(size_t length) {static const char data[] ="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";// std::random_device might actually be deterministic on some// platforms, but due to lack of support in the c++ standard library,// doing better requires either some ugly hacks or breaking portability.- std::random_device seed_gen;+ static std::random_device seed_gen;// Request 128 bits of entropy for initialization- std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()};- std::mt19937 engine(seed_sequence);+ static std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(),+ seed_gen()};- std::string result = "--cpp-httplib-multipart-data-";+ static std::mt19937 engine(seed_sequence);- for (auto i = 0; i < 16; i++) {+ std::string result;+ for (size_t i = 0; i < length; i++) {result += data[engine() % (sizeof(data) - 1)];}-return result;}+inline std::string make_multipart_data_boundary() {+ return "--cpp-httplib-multipart-data-" + detail::random_string(16);+}+inline bool is_multipart_boundary_chars_valid(const std::string &boundary) {auto valid = true;for (size_t i = 0; i < boundary.size(); i++) {@@ -4304,48 +4733,97 @@ serialize_multipart_formdata(const MultipartFormDataItems &items,body += item.content + serialize_multipart_formdata_item_end();}- if (finish) body += serialize_multipart_formdata_finish(boundary);+ if (finish) { body += serialize_multipart_formdata_finish(boundary); }return body;}-inline std::pair<size_t, size_t>-get_range_offset_and_length(const Request &req, size_t content_length,- size_t index) {- auto r = req.ranges[index];+inline bool range_error(Request &req, Response &res) {+ if (!req.ranges.empty() && 200 <= res.status && res.status < 300) {+ ssize_t contant_len = static_cast<ssize_t>(+ res.content_length_ ? res.content_length_ : res.body.size());- if (r.first == -1 && r.second == -1) {- return std::make_pair(0, content_length);- }+ ssize_t prev_first_pos = -1;+ ssize_t prev_last_pos = -1;+ size_t overwrapping_count = 0;- auto slen = static_cast<ssize_t>(content_length);+ // NOTE: The following Range check is based on '14.2. Range' in RFC 9110+ // 'HTTP Semantics' to avoid potential denial-of-service attacks.+ // https://www.rfc-editor.org/rfc/rfc9110#section-14.2- if (r.first == -1) {- r.first = (std::max)(static_cast<ssize_t>(0), slen - r.second);- r.second = slen - 1;+ // Too many ranges+ if (req.ranges.size() > CPPHTTPLIB_RANGE_MAX_COUNT) { return true; }++ for (auto &r : req.ranges) {+ auto &first_pos = r.first;+ auto &last_pos = r.second;++ if (first_pos == -1 && last_pos == -1) {+ first_pos = 0;+ last_pos = contant_len;+ }++ if (first_pos == -1) {+ first_pos = contant_len - last_pos;+ last_pos = contant_len - 1;+ }++ if (last_pos == -1) { last_pos = contant_len - 1; }++ // Range must be within content length+ if (!(0 <= first_pos && first_pos <= last_pos &&+ last_pos <= contant_len - 1)) {+ return true;+ }++ // Ranges must be in ascending order+ if (first_pos <= prev_first_pos) { return true; }++ // Request must not have more than two overlapping ranges+ if (first_pos <= prev_last_pos) {+ overwrapping_count++;+ if (overwrapping_count > 2) { return true; }+ }++ prev_first_pos = (std::max)(prev_first_pos, first_pos);+ prev_last_pos = (std::max)(prev_last_pos, last_pos);+ }}- if (r.second == -1) { r.second = slen - 1; }+ return false;+}++inline std::pair<size_t, size_t>+get_range_offset_and_length(Range r, size_t content_length) {+ (void)(content_length); // patch to get rid of "unused parameter" on release build+ assert(r.first != -1 && r.second != -1);+ assert(0 <= r.first && r.first < static_cast<ssize_t>(content_length));+ assert(r.first <= r.second &&+ r.second < static_cast<ssize_t>(content_length));+return std::make_pair(r.first, static_cast<size_t>(r.second - r.first) + 1);}-inline std::string make_content_range_header_field(size_t offset, size_t length,- size_t content_length) {+inline std::string make_content_range_header_field(+ const std::pair<size_t, size_t> &offset_and_length, size_t content_length) {+ auto st = offset_and_length.first;+ auto ed = st + offset_and_length.second - 1;+std::string field = "bytes ";- field += std::to_string(offset);+ field += std::to_string(st);field += "-";- field += std::to_string(offset + length - 1);+ field += std::to_string(ed);field += "/";field += std::to_string(content_length);return field;}template <typename SToken, typename CToken, typename Content>-bool process_multipart_ranges_data(const Request &req, Response &res,+bool process_multipart_ranges_data(const Request &req,const std::string &boundary,const std::string &content_type,- SToken stoken, CToken ctoken,- Content content) {+ size_t content_length, SToken stoken,+ CToken ctoken, Content content) {for (size_t i = 0; i < req.ranges.size(); i++) {ctoken("--");stoken(boundary);@@ -4356,50 +4834,51 @@ bool process_multipart_ranges_data(const Request &req, Response &res,ctoken("\r\n");}- auto offsets = get_range_offset_and_length(req, res.body.size(), i);- auto offset = offsets.first;- auto length = offsets.second;+ auto offset_and_length =+ get_range_offset_and_length(req.ranges[i], content_length);ctoken("Content-Range: ");- stoken(make_content_range_header_field(offset, length, res.body.size()));+ stoken(make_content_range_header_field(offset_and_length, content_length));ctoken("\r\n");ctoken("\r\n");- if (!content(offset, length)) { return false; }++ if (!content(offset_and_length.first, offset_and_length.second)) {+ return false;+ }ctoken("\r\n");}ctoken("--");stoken(boundary);- ctoken("--\r\n");+ ctoken("--");return true;}-inline bool make_multipart_ranges_data(const Request &req, Response &res,+inline void make_multipart_ranges_data(const Request &req, Response &res,const std::string &boundary,const std::string &content_type,+ size_t content_length,std::string &data) {- return process_multipart_ranges_data(- req, res, boundary, content_type,+ process_multipart_ranges_data(+ req, boundary, content_type, content_length,[&](const std::string &token) { data += token; },[&](const std::string &token) { data += token; },[&](size_t offset, size_t length) {- if (offset < res.body.size()) {- data += res.body.substr(offset, length);- return true;- }- return false;+ assert(offset + length <= content_length);+ data += res.body.substr(offset, length);+ return true;});}-inline size_t-get_multipart_ranges_data_length(const Request &req, Response &res,- const std::string &boundary,- const std::string &content_type) {+inline size_t get_multipart_ranges_data_length(const Request &req,+ const std::string &boundary,+ const std::string &content_type,+ size_t content_length) {size_t data_length = 0;process_multipart_ranges_data(- req, res, boundary, content_type,+ req, boundary, content_type, content_length,[&](const std::string &token) { data_length += token.size(); },[&](const std::string &token) { data_length += token.size(); },[&](size_t /*offset*/, size_t length) {@@ -4411,13 +4890,13 @@ get_multipart_ranges_data_length(const Request &req, Response &res,}template <typename T>-inline bool write_multipart_ranges_data(Stream &strm, const Request &req,- Response &res,- const std::string &boundary,- const std::string &content_type,- const T &is_shutting_down) {+inline bool+write_multipart_ranges_data(Stream &strm, const Request &req, Response &res,+ const std::string &boundary,+ const std::string &content_type,+ size_t content_length, const T &is_shutting_down) {return process_multipart_ranges_data(- req, res, boundary, content_type,+ req, boundary, content_type, content_length,[&](const std::string &token) { strm.write(token); },[&](const std::string &token) { strm.write(token); },[&](size_t offset, size_t length) {@@ -4426,18 +4905,6 @@ inline bool write_multipart_ranges_data(Stream &strm, const Request &req,});}-inline std::pair<size_t, size_t>-get_range_offset_and_length(const Request &req, const Response &res,- size_t index) {- auto r = req.ranges[index];-- if (r.second == -1) {- r.second = static_cast<ssize_t>(res.content_length_) - 1;- }-- return std::make_pair(r.first, r.second - r.first + 1);-}-inline bool expect_content(const Request &req) {if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" ||req.method == "PRI" || req.method == "DELETE") {@@ -4471,7 +4938,7 @@ inline std::string message_digest(const std::string &s, const EVP_MD *algo) {std::stringstream ss;for (auto i = 0u; i < hash_length; ++i) {ss << std::hex << std::setw(2) << std::setfill('0')- << (unsigned int)hash[i];+ << static_cast<unsigned int>(hash[i]);}return ss.str();@@ -4564,7 +5031,7 @@ inline bool retrieve_root_certs_from_keychain(CFObjectPtr<CFArrayRef> &certs) {inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) {auto result = false;- for (int i = 0; i < CFArrayGetCount(certs); ++i) {+ for (auto i = 0; i < CFArrayGetCount(certs); ++i) {const auto cert = reinterpret_cast<const __SecCertificate *>(CFArrayGetValueAtIndex(certs, i));@@ -4707,7 +5174,7 @@ inline bool parse_www_authenticate(const Response &res,s = s.substr(pos + 1);auto beg = std::sregex_iterator(s.begin(), s.end(), re);for (auto i = beg; i != std::sregex_iterator(); ++i) {- auto m = *i;+ const auto &m = *i;auto key = s.substr(static_cast<size_t>(m.position(1)),static_cast<size_t>(m.length(1)));auto val = m.length(2) > 0@@ -4724,20 +5191,6 @@ inline bool parse_www_authenticate(const Response &res,return false;}-// https://stackoverflow.com/questions/440133/how-do-i-create-a-random-alpha-numeric-string-in-c/440240#answer-440240-inline std::string random_string(size_t length) {- auto randchar = []() -> char {- const char charset[] = "0123456789"- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"- "abcdefghijklmnopqrstuvwxyz";- const size_t max_index = (sizeof(charset) - 1);- return charset[static_cast<size_t>(std::rand()) % max_index];- };- std::string str(length, 0);- std::generate_n(str.begin(), length, randchar);- return str;-}-class ContentProviderAdapter {public:explicit ContentProviderAdapter(@@ -4782,7 +5235,7 @@ inline void hosted_at(const std::string &hostname,const auto &addr =*reinterpret_cast<struct sockaddr_storage *>(rp->ai_addr);std::string ip;- int dummy = -1;+ auto dummy = -1;if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip,dummy)) {addrs.push_back(ip);@@ -4802,10 +5255,11 @@ inline std::string append_query_params(const std::string &path,}// Header utilities-inline std::pair<std::string, std::string> make_range_header(Ranges ranges) {+inline std::pair<std::string, std::string>+make_range_header(const Ranges &ranges) {std::string field = "bytes=";auto i = 0;- for (auto r : ranges) {+ for (const auto &r : ranges) {if (i != 0) { field += ", "; }if (r.first != -1) { field += std::to_string(r.first); }field += '-';@@ -4924,7 +5378,7 @@ inline void Response::set_redirect(const std::string &url, int stat) {if (300 <= stat && stat < 400) {this->status = stat;} else {- this->status = 302;+ this->status = StatusCode::Found_302;}}}@@ -4943,13 +5397,22 @@ inline void Response::set_content(const std::string &s,set_content(s.data(), s.size(), content_type);}+inline void Response::set_content(std::string &&s,+ const std::string &content_type) {+ body = std::move(s);++ auto rng = headers.equal_range("Content-Type");+ headers.erase(rng.first, rng.second);+ set_header("Content-Type", content_type);+}+inline void Response::set_content_provider(size_t in_length, const std::string &content_type, ContentProvider provider,ContentProviderResourceReleaser resource_releaser) {set_header("Content-Type", content_type);content_length_ = in_length;if (in_length > 0) { content_provider_ = std::move(provider); }- content_provider_resource_releaser_ = resource_releaser;+ content_provider_resource_releaser_ = std::move(resource_releaser);is_chunked_content_provider_ = false;}@@ -4959,7 +5422,7 @@ inline void Response::set_content_provider(set_header("Content-Type", content_type);content_length_ = 0;content_provider_ = detail::ContentProviderAdapter(std::move(provider));- content_provider_resource_releaser_ = resource_releaser;+ content_provider_resource_releaser_ = std::move(resource_releaser);is_chunked_content_provider_ = false;}@@ -4969,7 +5432,7 @@ inline void Response::set_chunked_content_provider(set_header("Content-Type", content_type);content_length_ = 0;content_provider_ = detail::ContentProviderAdapter(std::move(provider));- content_provider_resource_releaser_ = resource_releaser;+ content_provider_resource_releaser_ = std::move(resource_releaser);is_chunked_content_provider_ = true;}@@ -5010,7 +5473,7 @@ inline SocketStream::SocketStream(socket_t sock, time_t read_timeout_sec,write_timeout_sec_(write_timeout_sec),write_timeout_usec_(write_timeout_usec), read_buff_(read_buff_size_, 0) {}-inline SocketStream::~SocketStream() {}+inline SocketStream::~SocketStream() = default;inline bool SocketStream::is_readable() const {return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;@@ -5120,6 +5583,99 @@ inline socket_t BufferStream::socket() const { return 0; }inline const std::string &BufferStream::get_buffer() const { return buffer; }+inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) {+ // One past the last ending position of a path param substring+ std::size_t last_param_end = 0;++#ifndef CPPHTTPLIB_NO_EXCEPTIONS+ // Needed to ensure that parameter names are unique during matcher+ // construction+ // If exceptions are disabled, only last duplicate path+ // parameter will be set+ std::unordered_set<std::string> param_name_set;+#endif++ while (true) {+ const auto marker_pos = pattern.find(marker, last_param_end);+ if (marker_pos == std::string::npos) { break; }++ static_fragments_.push_back(+ pattern.substr(last_param_end, marker_pos - last_param_end));++ const auto param_name_start = marker_pos + 1;++ auto sep_pos = pattern.find(separator, param_name_start);+ if (sep_pos == std::string::npos) { sep_pos = pattern.length(); }++ auto param_name =+ pattern.substr(param_name_start, sep_pos - param_name_start);++#ifndef CPPHTTPLIB_NO_EXCEPTIONS+ if (param_name_set.find(param_name) != param_name_set.cend()) {+ std::string msg = "Encountered path parameter '" + param_name ++ "' multiple times in route pattern '" + pattern + "'.";+ throw std::invalid_argument(msg);+ }+#endif++ param_names_.push_back(std::move(param_name));++ last_param_end = sep_pos + 1;+ }++ if (last_param_end < pattern.length()) {+ static_fragments_.push_back(pattern.substr(last_param_end));+ }+}++inline bool PathParamsMatcher::match(Request &request) const {+ request.matches = std::smatch();+ request.path_params.clear();+ request.path_params.reserve(param_names_.size());++ // One past the position at which the path matched the pattern last time+ std::size_t starting_pos = 0;+ for (size_t i = 0; i < static_fragments_.size(); ++i) {+ const auto &fragment = static_fragments_[i];++ if (starting_pos + fragment.length() > request.path.length()) {+ return false;+ }++ // Avoid unnecessary allocation by using strncmp instead of substr ++ // comparison+ if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(),+ fragment.length()) != 0) {+ return false;+ }++ starting_pos += fragment.length();++ // Should only happen when we have a static fragment after a param+ // Example: '/users/:id/subscriptions'+ // The 'subscriptions' fragment here does not have a corresponding param+ if (i >= param_names_.size()) { continue; }++ auto sep_pos = request.path.find(separator, starting_pos);+ if (sep_pos == std::string::npos) { sep_pos = request.path.length(); }++ const auto ¶m_name = param_names_[i];++ request.path_params.emplace(+ param_name, request.path.substr(starting_pos, sep_pos - starting_pos));++ // Mark everythin up to '/' as matched+ starting_pos = sep_pos + 1;+ }+ // Returns false if the path is longer than the pattern+ return starting_pos >= request.path.length();+}++inline bool RegexMatcher::match(Request &request) const {+ request.path_params.clear();+ return std::regex_match(request.path, request.matches, regex_);+}+} // namespace detail// HTTP server implementation@@ -5131,69 +5687,72 @@ inline Server::Server()#endif}-inline Server::~Server() {}+inline Server::~Server() = default;++inline std::unique_ptr<detail::MatcherBase>+Server::make_matcher(const std::string &pattern) {+ if (pattern.find("/:") != std::string::npos) {+ return detail::make_unique<detail::PathParamsMatcher>(pattern);+ } else {+ return detail::make_unique<detail::RegexMatcher>(pattern);+ }+}inline Server &Server::Get(const std::string &pattern, Handler handler) {- get_handlers_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ get_handlers_.emplace_back(make_matcher(pattern), std::move(handler));return *this;}inline Server &Server::Post(const std::string &pattern, Handler handler) {- post_handlers_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ post_handlers_.emplace_back(make_matcher(pattern), std::move(handler));return *this;}inline Server &Server::Post(const std::string &pattern,HandlerWithContentReader handler) {- post_handlers_for_content_reader_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ post_handlers_for_content_reader_.emplace_back(make_matcher(pattern),+ std::move(handler));return *this;}inline Server &Server::Put(const std::string &pattern, Handler handler) {- put_handlers_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ put_handlers_.emplace_back(make_matcher(pattern), std::move(handler));return *this;}inline Server &Server::Put(const std::string &pattern,HandlerWithContentReader handler) {- put_handlers_for_content_reader_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ put_handlers_for_content_reader_.emplace_back(make_matcher(pattern),+ std::move(handler));return *this;}inline Server &Server::Patch(const std::string &pattern, Handler handler) {- patch_handlers_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ patch_handlers_.emplace_back(make_matcher(pattern), std::move(handler));return *this;}inline Server &Server::Patch(const std::string &pattern,HandlerWithContentReader handler) {- patch_handlers_for_content_reader_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ patch_handlers_for_content_reader_.emplace_back(make_matcher(pattern),+ std::move(handler));return *this;}inline Server &Server::Delete(const std::string &pattern, Handler handler) {- delete_handlers_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ delete_handlers_.emplace_back(make_matcher(pattern), std::move(handler));return *this;}inline Server &Server::Delete(const std::string &pattern,HandlerWithContentReader handler) {- delete_handlers_for_content_reader_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ delete_handlers_for_content_reader_.emplace_back(make_matcher(pattern),+ std::move(handler));return *this;}inline Server &Server::Options(const std::string &pattern, Handler handler) {- options_handlers_.push_back(- std::make_pair(std::regex(pattern), std::move(handler)));+ options_handlers_.emplace_back(make_matcher(pattern), std::move(handler));return *this;}@@ -5231,6 +5790,11 @@ Server::set_file_extension_and_mimetype_mapping(const std::string &ext,return *this;}+inline Server &Server::set_default_file_mimetype(const std::string &mime) {+ default_file_mimetype_ = mime;+ return *this;+}+inline Server &Server::set_file_request_handler(Handler handler) {file_request_handler_ = std::move(handler);return *this;@@ -5272,7 +5836,6 @@ inline Server &Server::set_logger(Logger logger) {inline Server &Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) {expect_100_continue_handler_ = std::move(handler);-return *this;}@@ -5296,6 +5859,12 @@ inline Server &Server::set_default_headers(Headers headers) {return *this;}+inline Server &Server::set_header_writer(+ std::function<ssize_t(Stream &, Headers &)> const &writer) {+ header_writer_ = writer;+ return *this;+}+inline Server &Server::set_keep_alive_max_count(size_t count) {keep_alive_max_count_ = count;return *this;@@ -5331,8 +5900,7 @@ inline Server &Server::set_payload_max_length(size_t length) {inline bool Server::bind_to_port(const std::string &host, int port,int socket_flags) {- if (bind_internal(host, port, socket_flags) < 0) return false;- return true;+ return bind_internal(host, port, socket_flags) >= 0;}inline int Server::bind_to_any_port(const std::string &host, int socket_flags) {return bind_internal(host, 0, socket_flags);@@ -5366,7 +5934,7 @@ inline void Server::stop() {}}-inline bool Server::parse_request_line(const char *s, Request &req) {+inline bool Server::parse_request_line(const char *s, Request &req) const {auto len = strlen(s);if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') { return false; }len -= 2;@@ -5407,7 +5975,7 @@ inline bool Server::parse_request_line(const char *s, Request &req) {size_t count = 0;detail::split(req.target.data(), req.target.data() + req.target.size(), '?',- [&](const char *b, const char *e) {+ 2, [&](const char *b, const char *e) {switch (count) {case 0:req.path = detail::decode_url(std::string(b, e), false);@@ -5430,7 +5998,10 @@ inline bool Server::parse_request_line(const char *s, Request &req) {}inline bool Server::write_response(Stream &strm, bool close_connection,- const Request &req, Response &res) {+ Request &req, Response &res) {+ // NOTE: `req.ranges` should be empty, otherwise it will be applied+ // incorrectly to the error content.+ req.ranges.clear();return write_response_core(strm, close_connection, req, res, false);}@@ -5486,11 +6057,11 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection,detail::BufferStream bstrm;if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status,- detail::status_message(res.status))) {+ status_message(res.status))) {return false;}- if (!detail::write_headers(bstrm, res.headers)) { return false; }+ if (!header_writer_(bstrm, res.headers)) { return false; }// Flush bufferauto &data = bstrm.get_buffer();@@ -5508,7 +6079,6 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection,if (write_content_with_provider(strm, req, res, boundary, content_type)) {res.content_provider_success_ = true;} else {- res.content_provider_success_ = false;ret = false;}}@@ -5533,15 +6103,16 @@ Server::write_content_with_provider(Stream &strm, const Request &req,return detail::write_content(strm, res.content_provider_, 0,res.content_length_, is_shutting_down);} else if (req.ranges.size() == 1) {- auto offsets =- detail::get_range_offset_and_length(req, res.content_length_, 0);- auto offset = offsets.first;- auto length = offsets.second;- return detail::write_content(strm, res.content_provider_, offset, length,- is_shutting_down);+ auto offset_and_length = detail::get_range_offset_and_length(+ req.ranges[0], res.content_length_);++ return detail::write_content(strm, res.content_provider_,+ offset_and_length.first,+ offset_and_length.second, is_shutting_down);} else {return detail::write_multipart_ranges_data(- strm, req, res, boundary, content_type, is_shutting_down);+ strm, req, res, boundary, content_type, res.content_length_,+ is_shutting_down);}} else {if (res.is_chunked_content_provider_) {@@ -5598,7 +6169,7 @@ inline bool Server::read_content(Stream &strm, Request &req, Response &res) {const auto &content_type = req.get_header_value("Content-Type");if (!content_type.find("application/x-www-form-urlencoded")) {if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) {- res.status = 413; // NOTE: should be 414?+ res.status = StatusCode::PayloadTooLarge_413; // NOTE: should be 414?return false;}detail::parse_query_text(req.body, req.params);@@ -5617,10 +6188,11 @@ inline bool Server::read_content_with_content_receiver(std::move(multipart_receiver));}-inline bool Server::read_content_core(Stream &strm, Request &req, Response &res,- ContentReceiver receiver,- MultipartContentHeader multipart_header,- ContentReceiver multipart_receiver) {+inline bool+Server::read_content_core(Stream &strm, Request &req, Response &res,+ ContentReceiver receiver,+ MultipartContentHeader multipart_header,+ ContentReceiver multipart_receiver) const {detail::MultipartFormDataParser multipart_form_data_parser;ContentReceiverWithProgress out;@@ -5628,7 +6200,7 @@ inline bool Server::read_content_core(Stream &strm, Request &req, Response &res,const auto &content_type = req.get_header_value("Content-Type");std::string boundary;if (!detail::parse_multipart_boundary(content_type, boundary)) {- res.status = 400;+ res.status = StatusCode::BadRequest_400;return false;}@@ -5664,7 +6236,7 @@ inline bool Server::read_content_core(Stream &strm, Request &req, Response &res,if (req.is_multipart_form_data()) {if (!multipart_form_data_parser.is_valid()) {- res.status = 400;+ res.status = StatusCode::BadRequest_400;return false;}}@@ -5683,17 +6255,26 @@ inline bool Server::handle_file_request(const Request &req, Response &res,if (path.back() == '/') { path += "index.html"; }if (detail::is_file(path)) {- detail::read_file(path, res.body);- auto type =- detail::find_content_type(path, file_extension_and_mimetype_map_);- if (type) { res.set_header("Content-Type", type); }for (const auto &kv : entry.headers) {- res.set_header(kv.first.c_str(), kv.second);+ res.set_header(kv.first, kv.second);}- res.status = req.has_header("Range") ? 206 : 200;++ auto mm = std::make_shared<detail::mmap>(path.c_str());+ if (!mm->is_open()) { return false; }++ res.set_content_provider(+ mm->size(),+ detail::find_content_type(path, file_extension_and_mimetype_map_,+ default_file_mimetype_),+ [mm](size_t offset, size_t length, DataSink &sink) -> bool {+ sink.write(mm->data() + offset, length);+ return true;+ });+if (!head && file_request_handler_) {file_request_handler_(req, res);}+return true;}}@@ -5789,13 +6370,14 @@ inline bool Server::listen_internal() {#ifdef _WIN32auto timeout = static_cast<uint32_t>(read_timeout_sec_ * 1000 +read_timeout_usec_ / 1000);- setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,- sizeof(timeout));+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO,+ reinterpret_cast<const char *>(&timeout), sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(read_timeout_sec_);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(read_timeout_usec_);- setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv));+ setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO,+ reinterpret_cast<const void *>(&tv), sizeof(tv));#endif}{@@ -5803,17 +6385,22 @@ inline bool Server::listen_internal() {#ifdef _WIN32auto timeout = static_cast<uint32_t>(write_timeout_sec_ * 1000 +write_timeout_usec_ / 1000);- setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout,- sizeof(timeout));+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO,+ reinterpret_cast<const char *>(&timeout), sizeof(timeout));#elsetimeval tv;tv.tv_sec = static_cast<long>(write_timeout_sec_);tv.tv_usec = static_cast<decltype(tv.tv_usec)>(write_timeout_usec_);- setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv));+ setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO,+ reinterpret_cast<const void *>(&tv), sizeof(tv));#endif}- task_queue->enqueue([this, sock]() { process_and_close_socket(sock); });+ if (!task_queue->enqueue(+ [this, sock]() { process_and_close_socket(sock); })) {+ detail::shutdown_socket(sock);+ detail::close_socket(sock);+ }}task_queue->shutdown();@@ -5829,7 +6416,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) {}// File handler- bool is_head_request = req.method == "HEAD";+ auto is_head_request = req.method == "HEAD";if ((req.method == "GET" || is_head_request) &&handle_file_request(req, res, is_head_request)) {return true;@@ -5895,17 +6482,17 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) {return dispatch_request(req, res, patch_handlers_);}- res.status = 400;+ res.status = StatusCode::BadRequest_400;return false;}inline bool Server::dispatch_request(Request &req, Response &res,- const Handlers &handlers) {+ const Handlers &handlers) const {for (const auto &x : handlers) {- const auto &pattern = x.first;+ const auto &matcher = x.first;const auto &handler = x.second;- if (std::regex_match(req.path, req.matches, pattern)) {+ if (matcher->match(req)) {handler(req, res);return true;}@@ -5915,18 +6502,18 @@ inline bool Server::dispatch_request(Request &req, Response &res,inline void Server::apply_ranges(const Request &req, Response &res,std::string &content_type,- std::string &boundary) {+ std::string &boundary) const {if (req.ranges.size() > 1) {- boundary = detail::make_multipart_data_boundary();-auto it = res.headers.find("Content-Type");if (it != res.headers.end()) {content_type = it->second;res.headers.erase(it);}- res.headers.emplace("Content-Type",- "multipart/byteranges; boundary=" + boundary);+ boundary = detail::make_multipart_data_boundary();++ res.set_header("Content-Type",+ "multipart/byteranges; boundary=" + boundary);}auto type = detail::encoding_type(req, res);@@ -5937,16 +6524,17 @@ inline void Server::apply_ranges(const Request &req, Response &res,if (req.ranges.empty()) {length = res.content_length_;} else if (req.ranges.size() == 1) {- auto offsets =- detail::get_range_offset_and_length(req, res.content_length_, 0);- auto offset = offsets.first;- length = offsets.second;+ auto offset_and_length = detail::get_range_offset_and_length(+ req.ranges[0], res.content_length_);++ length = offset_and_length.second;+auto content_range = detail::make_content_range_header_field(- offset, length, res.content_length_);+ offset_and_length, res.content_length_);res.set_header("Content-Range", content_range);} else {- length = detail::get_multipart_ranges_data_length(req, res, boundary,- content_type);+ length = detail::get_multipart_ranges_data_length(+ req, boundary, content_type, res.content_length_);}res.set_header("Content-Length", std::to_string(length));} else {@@ -5965,28 +6553,22 @@ inline void Server::apply_ranges(const Request &req, Response &res,if (req.ranges.empty()) {;} else if (req.ranges.size() == 1) {- auto offsets =- detail::get_range_offset_and_length(req, res.body.size(), 0);- auto offset = offsets.first;- auto length = offsets.second;+ auto offset_and_length =+ detail::get_range_offset_and_length(req.ranges[0], res.body.size());+ auto offset = offset_and_length.first;+ auto length = offset_and_length.second;+auto content_range = detail::make_content_range_header_field(- offset, length, res.body.size());+ offset_and_length, res.body.size());res.set_header("Content-Range", content_range);- if (offset < res.body.size()) {- res.body = res.body.substr(offset, length);- } else {- res.body.clear();- res.status = 416;- }++ assert(offset + length <= res.body.size());+ res.body = res.body.substr(offset, length);} else {std::string data;- if (detail::make_multipart_ranges_data(req, res, boundary, content_type,- data)) {- res.body.swap(data);- } else {- res.body.clear();- res.status = 416;- }+ detail::make_multipart_ranges_data(req, res, boundary, content_type,+ res.body.size(), data);+ res.body.swap(data);}if (type != detail::EncodingType::None) {@@ -6025,12 +6607,12 @@ inline void Server::apply_ranges(const Request &req, Response &res,inline bool Server::dispatch_request_for_content_reader(Request &req, Response &res, ContentReader content_reader,- const HandlersForContentReader &handlers) {+ const HandlersForContentReader &handlers) const {for (const auto &x : handlers) {- const auto &pattern = x.first;+ const auto &matcher = x.first;const auto &handler = x.second;- if (std::regex_match(req.path, req.matches, pattern)) {+ if (matcher->match(req)) {handler(req, res, content_reader);return true;}@@ -6050,15 +6632,10 @@ Server::process_request(Stream &strm, bool close_connection,if (!line_reader.getline()) { return false; }Request req;- Response res;+ Response res;res.version = "HTTP/1.1";-- for (const auto &header : default_headers_) {- if (res.headers.find(header.first) == res.headers.end()) {- res.headers.insert(header);- }- }+ res.headers = default_headers_;#ifdef _WIN32// TODO: Increase FD_SETSIZE statically (libzmq), dynamically (MySQL).@@ -6068,7 +6645,7 @@ Server::process_request(Stream &strm, bool close_connection,if (strm.socket() >= FD_SETSIZE) {Headers dummy;detail::read_headers(strm, dummy);- res.status = 500;+ res.status = StatusCode::InternalServerError_500;return write_response(strm, close_connection, req, res);}#endif@@ -6078,14 +6655,14 @@ Server::process_request(Stream &strm, bool close_connection,if (line_reader.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) {Headers dummy;detail::read_headers(strm, dummy);- res.status = 414;+ res.status = StatusCode::UriTooLong_414;return write_response(strm, close_connection, req, res);}// Request line and headersif (!parse_request_line(line_reader.ptr(), req) ||!detail::read_headers(strm, req.headers)) {- res.status = 400;+ res.status = StatusCode::BadRequest_400;return write_response(strm, close_connection, req, res);}@@ -6109,7 +6686,7 @@ Server::process_request(Stream &strm, bool close_connection,if (req.has_header("Range")) {const auto &range_header_value = req.get_header_value("Range");if (!detail::parse_range_header(range_header_value, req.ranges)) {- res.status = 416;+ res.status = StatusCode::RangeNotSatisfiable_416;return write_response(strm, close_connection, req, res);}}@@ -6117,22 +6694,22 @@ Server::process_request(Stream &strm, bool close_connection,if (setup_request) { setup_request(req); }if (req.get_header_value("Expect") == "100-continue") {- auto status = 100;+ int status = StatusCode::Continue_100;if (expect_100_continue_handler_) {status = expect_100_continue_handler_(req, res);}switch (status) {- case 100:- case 417:+ case StatusCode::Continue_100:+ case StatusCode::ExpectationFailed_417:strm.write_format("HTTP/1.1 %d %s\r\n\r\n", status,- detail::status_message(status));+ status_message(status));break;default: return write_response(strm, close_connection, req, res);}}- // Rounting- bool routed = false;+ // Routing+ auto routed = false;#ifdef CPPHTTPLIB_NO_EXCEPTIONSrouted = routing(req, res, strm);#else@@ -6144,7 +6721,7 @@ Server::process_request(Stream &strm, bool close_connection,exception_handler_(req, res, ep);routed = true;} else {- res.status = 500;+ res.status = StatusCode::InternalServerError_500;std::string val;auto s = e.what();for (size_t i = 0; s[i]; i++) {@@ -6162,17 +6739,29 @@ Server::process_request(Stream &strm, bool close_connection,exception_handler_(req, res, ep);routed = true;} else {- res.status = 500;+ res.status = StatusCode::InternalServerError_500;res.set_header("EXCEPTION_WHAT", "UNKNOWN");}}#endif-if (routed) {- if (res.status == -1) { res.status = req.ranges.empty() ? 200 : 206; }+ if (res.status == -1) {+ res.status = req.ranges.empty() ? StatusCode::OK_200+ : StatusCode::PartialContent_206;+ }++ if (detail::range_error(req, res)) {+ res.body.clear();+ res.content_length_ = 0;+ res.content_provider_ = nullptr;+ res.status = StatusCode::RangeNotSatisfiable_416;+ return write_response(strm, close_connection, req, res);+ }+return write_response_with_content(strm, close_connection, req, res);} else {- if (res.status == -1) { res.status = 404; }+ if (res.status == -1) { res.status = StatusCode::NotFound_404; }+return write_response(strm, close_connection, req, res);}}@@ -6272,7 +6861,7 @@ inline socket_t ClientImpl::create_client_socket(Error &error) const {// Check is custom IP specified for host_std::string ip;auto it = addr_map_.find(host_);- if (it != addr_map_.end()) ip = it->second;+ if (it != addr_map_.end()) { ip = it->second; }return detail::create_client_socket(host_, ip, port_, address_family_, tcp_nodelay_, socket_options_,@@ -6297,7 +6886,7 @@ inline void ClientImpl::shutdown_ssl(Socket & /*socket*/,socket_requests_are_from_thread_ == std::this_thread::get_id());}-inline void ClientImpl::shutdown_socket(Socket &socket) {+inline void ClientImpl::shutdown_socket(Socket &socket) const {if (socket.sock == INVALID_SOCKET) { return; }detail::shutdown_socket(socket.sock);}@@ -6322,7 +6911,7 @@ inline void ClientImpl::close_socket(Socket &socket) {}inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,- Response &res) {+ Response &res) const {std::array<char, 2048> buf{};detail::stream_line_reader line_reader(strm, buf.data(), buf.size());@@ -6330,9 +6919,9 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,if (!line_reader.getline()) { return false; }#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR- const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n");-#elseconst static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n");+#else+ const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n");#endifstd::cmatch m;@@ -6344,7 +6933,7 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,res.reason = std::string(m[3]);// Ignore '100 Continue'- while (res.status == 100) {+ while (res.status == StatusCode::Continue_100) {if (!line_reader.getline()) { return false; } // CRLFif (!line_reader.getline()) { return false; } // next response line@@ -6492,15 +7081,31 @@ inline bool ClientImpl::handle_request(Stream &strm, Request &req,if (!ret) { return false; }+ if (res.get_header_value("Connection") == "close" ||+ (res.version == "HTTP/1.0" && res.reason != "Connection established")) {+ // TODO this requires a not-entirely-obvious chain of calls to be correct+ // for this to be safe.++ // This is safe to call because handle_request is only called by send_+ // which locks the request mutex during the process. It would be a bug+ // to call it from a different thread since it's a thread-safety issue+ // to do these things to the socket if another thread is using the socket.+ std::lock_guard<std::mutex> guard(socket_mutex_);+ shutdown_ssl(socket_, true);+ shutdown_socket(socket_);+ close_socket(socket_);+ }+if (300 < res.status && res.status < 400 && follow_location_) {req = req_save;ret = redirect(req, res, error);}#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- if ((res.status == 401 || res.status == 407) &&+ if ((res.status == StatusCode::Unauthorized_401 ||+ res.status == StatusCode::ProxyAuthenticationRequired_407) &&req.authorization_count_ < 5) {- auto is_proxy = res.status == 407;+ auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407;const auto &username =is_proxy ? proxy_digest_auth_username_ : digest_auth_username_;const auto &password =@@ -6571,7 +7176,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {} else {if (next_scheme == "https") {#ifdef CPPHTTPLIB_OPENSSL_SUPPORT- SSLClient cli(next_host.c_str(), next_port);+ SSLClient cli(next_host, next_port);cli.copy_settings(*this);if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); }return detail::redirect(cli, req, res, path, location, error);@@ -6579,7 +7184,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {return false;#endif} else {- ClientImpl cli(next_host.c_str(), next_port);+ ClientImpl cli(next_host, next_port);cli.copy_settings(*this);return detail::redirect(cli, req, res, path, location, error);}@@ -6588,7 +7193,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {inline bool ClientImpl::write_content_with_provider(Stream &strm,const Request &req,- Error &error) {+ Error &error) const {auto is_shutting_down = []() { return false; };if (req.is_chunked_content_provider_) {@@ -6616,32 +7221,32 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,// Prepare additional headersif (close_connection) {if (!req.has_header("Connection")) {- req.headers.emplace("Connection", "close");+ req.set_header("Connection", "close");}}if (!req.has_header("Host")) {if (is_ssl()) {if (port_ == 443) {- req.headers.emplace("Host", host_);+ req.set_header("Host", host_);} else {- req.headers.emplace("Host", host_and_port_);+ req.set_header("Host", host_and_port_);}} else {if (port_ == 80) {- req.headers.emplace("Host", host_);+ req.set_header("Host", host_);} else {- req.headers.emplace("Host", host_and_port_);+ req.set_header("Host", host_and_port_);}}}- if (!req.has_header("Accept")) { req.headers.emplace("Accept", "*/*"); }+ if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); }#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENTif (!req.has_header("User-Agent")) {auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION;- req.headers.emplace("User-Agent", agent);+ req.set_header("User-Agent", agent);}#endif@@ -6650,23 +7255,23 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,if (!req.is_chunked_content_provider_) {if (!req.has_header("Content-Length")) {auto length = std::to_string(req.content_length_);- req.headers.emplace("Content-Length", length);+ req.set_header("Content-Length", length);}}} else {if (req.method == "POST" || req.method == "PUT" ||req.method == "PATCH") {- req.headers.emplace("Content-Length", "0");+ req.set_header("Content-Length", "0");}}} else {if (!req.has_header("Content-Type")) {- req.headers.emplace("Content-Type", "text/plain");+ req.set_header("Content-Type", "text/plain");}if (!req.has_header("Content-Length")) {auto length = std::to_string(req.body.size());- req.headers.emplace("Content-Length", length);+ req.set_header("Content-Length", length);}}@@ -6706,7 +7311,7 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,const auto &path = url_encode_ ? detail::encode_url(req.path) : req.path;bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str());- detail::write_headers(bstrm, req.headers);+ header_writer_(bstrm, req.headers);// Flush bufferauto &data = bstrm.get_buffer();@@ -6734,12 +7339,10 @@ inline std::unique_ptr<Response> ClientImpl::send_with_content_provider(ContentProvider content_provider,ContentProviderWithoutLength content_provider_without_length,const std::string &content_type, Error &error) {- if (!content_type.empty()) {- req.headers.emplace("Content-Type", content_type);- }+ if (!content_type.empty()) { req.set_header("Content-Type", content_type); }#ifdef CPPHTTPLIB_ZLIB_SUPPORT- if (compress_) { req.headers.emplace("Content-Encoding", "gzip"); }+ if (compress_) { req.set_header("Content-Encoding", "gzip"); }#endif#ifdef CPPHTTPLIB_ZLIB_SUPPORT@@ -6800,10 +7403,9 @@ inline std::unique_ptr<Response> ClientImpl::send_with_content_provider(req.content_provider_ = detail::ContentProviderAdapter(std::move(content_provider_without_length));req.is_chunked_content_provider_ = true;- req.headers.emplace("Transfer-Encoding", "chunked");+ req.set_header("Transfer-Encoding", "chunked");} else {req.body.assign(body, content_length);- ;}}@@ -6864,7 +7466,8 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,}// Body- if ((res.status != 204) && req.method != "HEAD" && req.method != "CONNECT") {+ if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" &&+ req.method != "CONNECT") {auto redirect = 300 < res.status && res.status < 400 && follow_location_;if (req.response_handler && !redirect) {@@ -6909,24 +7512,6 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,}}- if (res.get_header_value("Connection") == "close" ||- (res.version == "HTTP/1.0" && res.reason != "Connection established")) {- // TODO this requires a not-entirely-obvious chain of calls to be correct- // for this to be safe. Maybe a code refactor (such as moving this out to- // the send function and getting rid of the recursiveness of the mutex)- // could make this more obvious.-- // This is safe to call because process_request is only called by- // handle_request which is only called by send, which locks the request- // mutex during the process. It would be a bug to call it from a different- // thread since it's a thread-safety issue to do these things to the socket- // if another thread is using the socket.- std::lock_guard<std::mutex> guard(socket_mutex_);- shutdown_ssl(socket_, true);- shutdown_socket(socket_);- close_socket(socket_);- }-// Logif (logger_) { logger_(req, res); }@@ -6935,13 +7520,14 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(const std::string &boundary, const MultipartFormDataItems &items,- const MultipartFormDataProviderItems &provider_items) {- size_t cur_item = 0, cur_start = 0;+ const MultipartFormDataProviderItems &provider_items) const {+ size_t cur_item = 0;+ size_t cur_start = 0;// cur_item and cur_start are copied to within the std::function and maintain// state between successive callsreturn [&, cur_item, cur_start](size_t offset,DataSink &sink) mutable -> bool {- if (!offset && items.size()) {+ if (!offset && !items.empty()) {sink.os << detail::serialize_multipart_formdata(items, boundary, false);return true;} else if (cur_item < provider_items.size()) {@@ -6954,12 +7540,13 @@ inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(}DataSink cur_sink;- bool has_data = true;+ auto has_data = true;cur_sink.write = sink.write;cur_sink.done = [&]() { has_data = false; };- if (!provider_items[cur_item].provider(offset - cur_start, cur_sink))+ if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) {return false;+ }if (!has_data) {sink.os << detail::serialize_multipart_formdata_item_end();@@ -7078,14 +7665,15 @@ inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,if (params.empty()) { return Get(path, headers); }std::string path_with_query = append_query_params(path, params);- return Get(path_with_query.c_str(), headers, progress);+ return Get(path_with_query, headers, std::move(progress));}inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,const Headers &headers,ContentReceiver content_receiver,Progress progress) {- return Get(path, params, headers, nullptr, content_receiver, progress);+ return Get(path, params, headers, nullptr, std::move(content_receiver),+ std::move(progress));}inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,@@ -7094,12 +7682,13 @@ inline Result ClientImpl::Get(const std::string &path, const Params ¶ms,ContentReceiver content_receiver,Progress progress) {if (params.empty()) {- return Get(path, headers, response_handler, content_receiver, progress);+ return Get(path, headers, std::move(response_handler),+ std::move(content_receiver), std::move(progress));}std::string path_with_query = append_query_params(path, params);- return Get(path_with_query.c_str(), headers, response_handler,- content_receiver, progress);+ return Get(path_with_query, headers, std::move(response_handler),+ std::move(content_receiver), std::move(progress));}inline Result ClientImpl::Head(const std::string &path) {@@ -7201,7 +7790,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,const auto &content_type =detail::serialize_multipart_formdata_get_content_type(boundary);const auto &body = detail::serialize_multipart_formdata(items, boundary);- return Post(path, headers, body, content_type.c_str());+ return Post(path, headers, body, content_type);}inline Result ClientImpl::Post(const std::string &path, const Headers &headers,@@ -7214,7 +7803,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,const auto &content_type =detail::serialize_multipart_formdata_get_content_type(boundary);const auto &body = detail::serialize_multipart_formdata(items, boundary);- return Post(path, headers, body, content_type.c_str());+ return Post(path, headers, body, content_type);}inline Result@@ -7423,9 +8012,7 @@ inline Result ClientImpl::Delete(const std::string &path,req.headers = headers;req.path = path;- if (!content_type.empty()) {- req.headers.emplace("Content-Type", content_type);- }+ if (!content_type.empty()) { req.set_header("Content-Type", content_type); }req.body.assign(body, content_length);return send_(std::move(req));@@ -7458,13 +8045,6 @@ inline Result ClientImpl::Options(const std::string &path,return send_(std::move(req));}-inline size_t ClientImpl::is_socket_open() const {- std::lock_guard<std::mutex> guard(socket_mutex_);- return socket_.is_open();-}--inline socket_t ClientImpl::socket() const { return socket_.sock; }-inline void ClientImpl::stop() {std::lock_guard<std::mutex> guard(socket_mutex_);@@ -7488,6 +8068,17 @@ inline void ClientImpl::stop() {close_socket(socket_);}+inline std::string ClientImpl::host() const { return host_; }++inline int ClientImpl::port() const { return port_; }++inline size_t ClientImpl::is_socket_open() const {+ std::lock_guard<std::mutex> guard(socket_mutex_);+ return socket_.is_open();+}++inline socket_t ClientImpl::socket() const { return socket_.sock; }+inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) {connection_timeout_sec_ = sec;connection_timeout_usec_ = usec;@@ -7536,6 +8127,11 @@ inline void ClientImpl::set_default_headers(Headers headers) {default_headers_ = std::move(headers);}+inline void ClientImpl::set_header_writer(+ std::function<ssize_t(Stream &, Headers &)> const &writer) {+ header_writer_ = writer;+}+inline void ClientImpl::set_address_family(int family) {address_family_ = family;}@@ -7575,9 +8171,7 @@ inline void ClientImpl::set_proxy_digest_auth(const std::string &username,proxy_digest_auth_username_ = username;proxy_digest_auth_password_ = password;}-#endif-#ifdef CPPHTTPLIB_OPENSSL_SUPPORTinline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path,const std::string &ca_cert_dir_path) {ca_cert_file_path_ = ca_cert_file_path;@@ -7589,9 +8183,34 @@ inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) {ca_cert_store_ = ca_cert_store;}}-#endif-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT+inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert,+ std::size_t size) const {+ auto mem = BIO_new_mem_buf(ca_cert, static_cast<int>(size));+ if (!mem) { return nullptr; }++ auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr);+ if (!inf) {+ BIO_free_all(mem);+ return nullptr;+ }++ auto cts = X509_STORE_new();+ if (cts) {+ for (auto i = 0; i < static_cast<int>(sk_X509_INFO_num(inf)); i++) {+ auto itmp = sk_X509_INFO_value(inf, i);+ if (!itmp) { continue; }++ if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); }+ if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); }+ }+ }++ sk_X509_INFO_pop_free(inf, X509_INFO_free);+ BIO_free_all(mem);+ return cts;+}+inline void ClientImpl::enable_server_certificate_verification(bool enabled) {server_certificate_verification_ = enabled;}@@ -7655,7 +8274,7 @@ bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl,U ssl_connect_or_accept,time_t timeout_sec,time_t timeout_usec) {- int res = 0;+ auto res = 0;while ((res = ssl_connect_or_accept(ssl)) != 1) {auto err = SSL_get_error(ssl, res);switch (err) {@@ -7718,7 +8337,7 @@ inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl,SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY);}-inline SSLSocketStream::~SSLSocketStream() {}+inline SSLSocketStream::~SSLSocketStream() = default;inline bool SSLSocketStream::is_readable() const {return detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;@@ -7736,7 +8355,7 @@ inline ssize_t SSLSocketStream::read(char *ptr, size_t size) {auto ret = SSL_read(ssl_, ptr, static_cast<int>(size));if (ret < 0) {auto err = SSL_get_error(ssl_, ret);- int n = 1000;+ auto n = 1000;#ifdef _WIN32while (--n >= 0 && (err == SSL_ERROR_WANT_READ ||(err == SSL_ERROR_SYSCALL &&@@ -7769,7 +8388,7 @@ inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) {auto ret = SSL_write(ssl_, ptr, static_cast<int>(handle_size));if (ret < 0) {auto err = SSL_get_error(ssl_, ret);- int n = 1000;+ auto n = 1000;#ifdef _WIN32while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE ||(err == SSL_ERROR_SYSCALL &&@@ -7822,10 +8441,10 @@ inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path,SSL_CTX_set_min_proto_version(ctx_, TLS1_1_VERSION);- // add default password callback before opening encrypted private keyif (private_key_password != nullptr && (private_key_password[0] != '\0')) {- SSL_CTX_set_default_passwd_cb_userdata(ctx_,- (char *)private_key_password);+ SSL_CTX_set_default_passwd_cb_userdata(+ ctx_,+ reinterpret_cast<void *>(const_cast<char *>(private_key_password)));}if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 ||@@ -7927,16 +8546,23 @@ inline SSLClient::SSLClient(const std::string &host, int port)inline SSLClient::SSLClient(const std::string &host, int port,const std::string &client_cert_path,- const std::string &client_key_path)+ const std::string &client_key_path,+ const std::string &private_key_password): ClientImpl(host, port, client_cert_path, client_key_path) {ctx_ = SSL_CTX_new(TLS_client_method());detail::split(&host_[0], &host_[host_.size()], '.',[&](const char *b, const char *e) {- host_components_.emplace_back(std::string(b, e));+ host_components_.emplace_back(b, e);});if (!client_cert_path.empty() && !client_key_path.empty()) {+ if (!private_key_password.empty()) {+ SSL_CTX_set_default_passwd_cb_userdata(+ ctx_, reinterpret_cast<void *>(+ const_cast<char *>(private_key_password.c_str())));+ }+if (SSL_CTX_use_certificate_file(ctx_, client_cert_path.c_str(),SSL_FILETYPE_PEM) != 1 ||SSL_CTX_use_PrivateKey_file(ctx_, client_key_path.c_str(),@@ -7948,16 +8574,23 @@ inline SSLClient::SSLClient(const std::string &host, int port,}inline SSLClient::SSLClient(const std::string &host, int port,- X509 *client_cert, EVP_PKEY *client_key)+ X509 *client_cert, EVP_PKEY *client_key,+ const std::string &private_key_password): ClientImpl(host, port) {ctx_ = SSL_CTX_new(TLS_client_method());detail::split(&host_[0], &host_[host_.size()], '.',[&](const char *b, const char *e) {- host_components_.emplace_back(std::string(b, e));+ host_components_.emplace_back(b, e);});if (client_cert != nullptr && client_key != nullptr) {+ if (!private_key_password.empty()) {+ SSL_CTX_set_default_passwd_cb_userdata(+ ctx_, reinterpret_cast<void *>(+ const_cast<char *>(private_key_password.c_str())));+ }+if (SSL_CTX_use_certificate(ctx_, client_cert) != 1 ||SSL_CTX_use_PrivateKey(ctx_, client_key) != 1) {SSL_CTX_free(ctx_);@@ -7989,6 +8622,11 @@ inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) {}}+inline void SSLClient::load_ca_cert_store(const char *ca_cert,+ std::size_t size) {+ set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size));+}+inline long SSLClient::get_openssl_verify_result() const {return verify_result_;}@@ -8003,14 +8641,14 @@ inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) {inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,bool &success, Error &error) {success = true;- Response res2;+ Response proxy_res;if (!detail::process_client_socket(socket.sock, read_timeout_sec_, read_timeout_usec_,write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {Request req2;req2.method = "CONNECT";req2.path = host_and_port_;- return process_request(strm, req2, res2, false, error);+ return process_request(strm, req2, proxy_res, false, error);})) {// Thread-safe to close everything because we are assuming there are no// requests in flight@@ -8021,12 +8659,12 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,return false;}- if (res2.status == 407) {+ if (proxy_res.status == StatusCode::ProxyAuthenticationRequired_407) {if (!proxy_digest_auth_username_.empty() &&!proxy_digest_auth_password_.empty()) {std::map<std::string, std::string> auth;- if (detail::parse_www_authenticate(res2, auth, true)) {- Response res3;+ if (detail::parse_www_authenticate(proxy_res, auth, true)) {+ proxy_res = Response();if (!detail::process_client_socket(socket.sock, read_timeout_sec_, read_timeout_usec_,write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) {@@ -8037,7 +8675,7 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,req3, auth, 1, detail::random_string(10),proxy_digest_auth_username_, proxy_digest_auth_password_,true));- return process_request(strm, req3, res3, false, error);+ return process_request(strm, req3, proxy_res, false, error);})) {// Thread-safe to close everything because we are assuming there are// no requests in flight@@ -8048,17 +8686,28 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res,return false;}}- } else {- res = res2;- return false;}}+ // If status code is not 200, proxy request is failed.+ // Set error to ProxyConnection and return proxy response+ // as the response of the request+ if (proxy_res.status != StatusCode::OK_200) {+ error = Error::ProxyConnection;+ res = std::move(proxy_res);+ // Thread-safe to close everything because we are assuming there are+ // no requests in flight+ shutdown_ssl(socket, true);+ shutdown_socket(socket);+ close_socket(socket);+ return false;+ }+return true;}inline bool SSLClient::load_certs() {- bool ret = true;+ auto ret = true;std::call_once(initialize_cert_, [&]() {std::lock_guard<std::mutex> guard(ctx_mutex_);@@ -8134,7 +8783,11 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) {return true;},[&](SSL *ssl2) {- SSL_set_tlsext_host_name(ssl2, host_.c_str());+ // NOTE: Direct call instead of using the OpenSSL macro to suppress+ // -Wold-style-cast warning+ // SSL_set_tlsext_host_name(ssl2, host_.c_str());+ SSL_ctrl(ssl2, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name,+ static_cast<void *>(const_cast<char *>(host_.c_str())));return true;});@@ -8208,8 +8861,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {auto type = GEN_DNS;- struct in6_addr addr6;- struct in_addr addr;+ struct in6_addr addr6 {};+ struct in_addr addr {};size_t addr_len = 0;#ifndef __MINGW32__@@ -8234,8 +8887,9 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {for (decltype(count) i = 0; i < count && !dsn_matched; i++) {auto val = sk_GENERAL_NAME_value(alt_names, i);if (val->type == type) {- auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5);- auto name_len = (size_t)ASN1_STRING_length(val->d.ia5);+ auto name =+ reinterpret_cast<const char *>(ASN1_STRING_get0_data(val->d.ia5));+ auto name_len = static_cast<size_t>(ASN1_STRING_length(val->d.ia5));switch (type) {case GEN_DNS: dsn_matched = check_host_name(name, name_len); break;@@ -8253,7 +8907,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {if (dsn_matched || ip_matched) { ret = true; }}- GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names);+ GENERAL_NAMES_free(const_cast<STACK_OF(GENERAL_NAME) *>(+ reinterpret_cast<const STACK_OF(GENERAL_NAME) *>(alt_names)));return ret;}@@ -8282,7 +8937,7 @@ inline bool SSLClient::check_host_name(const char *pattern,std::vector<std::string> pattern_components;detail::split(&pattern[0], &pattern[pattern_len], '.',[&](const char *b, const char *e) {- pattern_components.emplace_back(std::string(b, e));+ pattern_components.emplace_back(b, e);});if (host_components_.size() != pattern_components.size()) { return false; }@@ -8361,7 +9016,7 @@ inline Client::Client(const std::string &host, int port,: cli_(detail::make_unique<ClientImpl>(host, port, client_cert_path,client_key_path)) {}-inline Client::~Client() {}+inline Client::~Client() = default;inline bool Client::is_valid() const {return cli_ != nullptr && cli_->is_valid();@@ -8421,19 +9076,20 @@ inline Result Client::Get(const std::string &path, const Headers &headers,}inline Result Client::Get(const std::string &path, const Params ¶ms,const Headers &headers, Progress progress) {- return cli_->Get(path, params, headers, progress);+ return cli_->Get(path, params, headers, std::move(progress));}inline Result Client::Get(const std::string &path, const Params ¶ms,const Headers &headers,ContentReceiver content_receiver, Progress progress) {- return cli_->Get(path, params, headers, content_receiver, progress);+ return cli_->Get(path, params, headers, std::move(content_receiver),+ std::move(progress));}inline Result Client::Get(const std::string &path, const Params ¶ms,const Headers &headers,ResponseHandler response_handler,ContentReceiver content_receiver, Progress progress) {- return cli_->Get(path, params, headers, response_handler, content_receiver,- progress);+ return cli_->Get(path, params, headers, std::move(response_handler),+ std::move(content_receiver), std::move(progress));}inline Result Client::Head(const std::string &path) { return cli_->Head(path); }@@ -8665,12 +9321,16 @@ inline bool Client::send(Request &req, Response &res, Error &error) {inline Result Client::send(const Request &req) { return cli_->send(req); }+inline void Client::stop() { cli_->stop(); }++inline std::string Client::host() const { return cli_->host(); }++inline int Client::port() const { return cli_->port(); }+inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); }inline socket_t Client::socket() const { return cli_->socket(); }-inline void Client::stop() { cli_->stop(); }-inline voidClient::set_hostname_addr_map(std::map<std::string, std::string> addr_map) {cli_->set_hostname_addr_map(std::move(addr_map));@@ -8680,6 +9340,11 @@ inline void Client::set_default_headers(Headers headers) {cli_->set_default_headers(std::move(headers));}+inline void Client::set_header_writer(+ std::function<ssize_t(Stream &, Headers &)> const &writer) {+ cli_->set_header_writer(writer);+}+inline void Client::set_address_family(int family) {cli_->set_address_family(family);}@@ -8754,7 +9419,9 @@ inline void Client::enable_server_certificate_verification(bool enabled) {}#endif-inline void Client::set_logger(Logger logger) { cli_->set_logger(logger); }+inline void Client::set_logger(Logger logger) {+ cli_->set_logger(std::move(logger));+}#ifdef CPPHTTPLIB_OPENSSL_SUPPORTinline void Client::set_ca_cert_path(const std::string &ca_cert_file_path,@@ -8770,6 +9437,10 @@ inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) {}}+inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) {+ set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size));+}+inline long Client::get_openssl_verify_result() const {if (is_ssl_) {return static_cast<SSLClient &>(*cli_).get_openssl_verify_result();diff --git a/llama.cpp/server/oai.h b/llama.cpp/server/oai.hdeleted file mode 100644index 1ddca949..00000000--- a/llama.cpp/server/oai.h+++ /dev/null@@ -1,228 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi--#pragma once--#include <string>-#include <vector>-#include <set>-#include <mutex>-#include <condition_variable>-#include <unordered_map>--#include "json.h"-#include "utils.h"--#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"--using json = nlohmann::json;--inline static json oaicompat_completion_params_parse(- const struct llama_model * model,- const json &body, /* openai api json semantics */- const std::string &chat_template)-{- json llama_params;-- llama_params["__oaicompat"] = true;-- // Map OpenAI parameters to llama.cpp parameters- //- // For parameters that are defined by the OpenAI documentation (e.g.- // temperature), we explicitly specify OpenAI's intended default; we- // need to do that because sometimes OpenAI disagrees with llama.cpp- //- // https://platform.openai.com/docs/api-reference/chat/create- llama_sampling_params default_sparams;- llama_params["model"] = json_value(body, "model", std::string("unknown"));- llama_params["prompt"] = format_chat(model, chat_template, body["messages"]);- llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);- llama_params["temperature"] = json_value(body, "temperature", 0.0);- llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);- llama_params["top_p"] = json_value(body, "top_p", 1.0);- llama_params["n_predict"] = json_value(body, "max_tokens", -1);- llama_params["logit_bias"] = json_value(body, "logit_bias",json::object());- llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);- llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);- llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);- llama_params["stream"] = json_value(body, "stream", false);- llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);- llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);- llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);- llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);- llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);- llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);- llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);- llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);-- if (body.count("grammar") != 0) {- llama_params["grammar"] = json_value(body, "grammar", json::object());- }-- // Handle 'stop' field- if (body.contains("stop") && body["stop"].is_string()) {- llama_params["stop"] = json::array({body["stop"].get<std::string>()});- } else {- llama_params["stop"] = json_value(body, "stop", json::array());- }-- // Ensure there is ChatML-specific end sequence among stop words- llama_params["stop"].push_back("<|im_end|>");-- return llama_params;-}--inline static json format_final_response_oaicompat(const json &request, const task_result &response, bool streaming = false)-{- json result = response.result_json;-- bool stopped_word = result.count("stopped_word") != 0;- bool stopped_eos = json_value(result, "stopped_eos", false);- int num_tokens_predicted = json_value(result, "tokens_predicted", 0);- int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);- std::string content = json_value(result, "content", std::string(""));-- std::string finish_reason = "length";- if (stopped_word || stopped_eos) {- finish_reason = "stop";- }-- json choices =- streaming ? json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"delta", json::object()}}})- : json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"message", json{{"content", content},- {"role", "assistant"}}}}});-- std::time_t t = std::time(0);-- json res =- json{{"choices", choices},- {"created", t},- {"model",- json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},- {"object", streaming ? "chat.completion.chunk" : "chat.completion"},- {"usage",- json{{"completion_tokens", num_tokens_predicted},- {"prompt_tokens", num_prompt_tokens},- {"total_tokens", num_tokens_predicted + num_prompt_tokens}}},- {"id", gen_chatcmplid()}};-- if (server_verbose) {- res["__verbose"] = result;- }-- if (result.contains("completion_probabilities")) {- res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());- }-- return res;-}--// return value is vector as there is one case where we might need to generate two responses-inline static std::vector<json> format_partial_response_oaicompat(const task_result &response) {- json result = response.result_json;-- if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {- return std::vector<json>({response.result_json});- }-- bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;- std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));-- bool stopped_word = json_value(result, "stopped_word", false);- bool stopped_eos = json_value(result, "stopped_eos", false);- bool stopped_limit = json_value(result, "stopped_limit", false);- std::string content = json_value(result, "content", std::string(""));-- std::string finish_reason;- if (stopped_word || stopped_eos) {- finish_reason = "stop";- }- if (stopped_limit) {- finish_reason = "length";- }-- std::time_t t = std::time(0);-- json choices;-- if (!finish_reason.empty()) {- choices = json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"delta", json::object()}}});- } else {- if (first) {- if (content.empty()) {- choices = json::array({json{{"finish_reason", nullptr},- {"index", 0},- {"delta", json{{"role", "assistant"}}}}});- } else {- // We have to send this as two updates to conform to openai behavior- json initial_ret = json{{"choices", json::array({json{- {"finish_reason", nullptr},- {"index", 0},- {"delta", json{- {"role", "assistant"}- }}}})},- {"created", t},- {"id", gen_chatcmplid()},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- json second_ret = json{- {"choices", json::array({json{{"finish_reason", nullptr},- {"index", 0},- {"delta", json{- {"content", content}}}- }})},- {"created", t},- {"id", gen_chatcmplid()},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- return std::vector<json>({initial_ret, second_ret});- }- } else {- // Some idiosyncrasy in task processing logic makes several trailing calls- // with empty content, we ignore these at the calee site.- if (content.empty()) {- return std::vector<json>({json::object()});- }-- choices = json::array({json{- {"finish_reason", nullptr},- {"index", 0},- {"delta",- json{- {"content", content},- }},- }});- }- }-- json ret = json{{"choices", choices},- {"created", t},- {"id", gen_chatcmplid()},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- return std::vector<json>({ret});-}--inline static json format_embeddings_response_oaicompat(const json &request, const json &embeddings)-{- json res =- json{- {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},- {"object", "list"},- {"usage",- json{{"prompt_tokens", 0},- {"total_tokens", 0}}},- {"data", embeddings}- };- return res;-}-diff --git a/llama.cpp/server/public/completion.js b/llama.cpp/server/public/completion.jsindex ab38a7b4..835ce6e6 100644--- a/llama.cpp/server/public/completion.js+++ b/llama.cpp/server/public/completion.js@@ -96,18 +96,18 @@ export async function* llama(prompt, params = {}, config = {}) {}}if (result.error) {- result.error = JSON.parse(result.error);- if (result.error.content.includes('slot unavailable')) {- // Throw an error to be caught by upstream callers- throw new Error('slot unavailable');- } else {- console.error(`llama.cpp error: ${result.error.content}`);+ try {+ result.error = JSON.parse(result.error);+ if (result.error.message.includes('slot unavailable')) {+ // Throw an error to be caught by upstream callers+ throw new Error('slot unavailable');+ } else {+ console.error(`llama.cpp error [${result.error.code} - ${result.error.type}]: ${result.error.message}`);+ }+ } catch(e) {+ console.error(`llama.cpp error ${result.error}`)}}- if (result.error) {- result.error = JSON.parse(result.error);- console.error(`llama.cpp error: ${result.error.content}`);- }}}}diff --git a/llama.cpp/server/public/index.html b/llama.cpp/server/public/index.htmlindex 84038ddc..18f95f19 100644--- a/llama.cpp/server/public/index.html+++ b/llama.cpp/server/public/index.html@@ -630,14 +630,16 @@const grammarJsonSchemaPropOrder = signal('')const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value- const convertJSONSchemaGrammar = () => {+ const convertJSONSchemaGrammar = async () => {try {- const schema = JSON.parse(params.value.grammar)- const converter = new SchemaConverter(- grammarJsonSchemaPropOrder.value+ let schema = JSON.parse(params.value.grammar)+ const converter = new SchemaConverter({+ prop_order: grammarJsonSchemaPropOrder.value.split(',')- .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {})- )+ .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),+ allow_fetch: true,+ })+ schema = await converter.resolveRefs(schema, 'input')converter.visit(schema, '')params.value = {...params.value,@@ -1018,7 +1020,7 @@<footer><p><${ModelGenerationInfo} /></p>- <p>Powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>+ <p>Powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>, and <a href="https://github.com/Mozilla-Ocho/llamafile/">llamafile</a></p></footer></div>`;diff --git a/llama.cpp/server/public/index.js b/llama.cpp/server/public/index.jsindex 9db5a9d9..aa3ffb5e 100644--- a/llama.cpp/server/public/index.js+++ b/llama.cpp/server/public/index.js@@ -1 +1 @@-function t(){throw new Error("Cycle detected")}function n(){if(u>1){u--;return}let t,n=!1;while(void 0!==_){let i=_;_=void 0;f++;while(void 0!==i){const _=i.o;i.o=void 0;i.f&=-3;if(!(8&i.f)&&a(i))try{i.c()}catch(e){if(!n){t=e;n=!0}}i=_}}f=0;u--;if(n)throw t}function e(t){if(u>0)return t();u++;try{return t()}finally{n()}}let i,_,o=0;function r(t){if(o>0)return t();const n=i;i=void 0;o++;try{return t()}finally{o--;i=n}}let u=0,f=0,l=0;function s(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function c(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}c.prototype.h=function(){return!0};c.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};c.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};c.prototype.subscribe=function(t){const n=this;return S((function(){const e=n.value,i=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=i}}))};c.prototype.valueOf=function(){return this.value};c.prototype.toString=function(){return this.value+""};c.prototype.toJSON=function(){return this.value};c.prototype.peek=function(){return this.v};Object.defineProperty(c.prototype,"value",{get(){const t=s(this);if(void 0!==t)t.i=this.i;return this.v},set(e){if(i instanceof v)!function(){throw new Error("Computed cannot have side-effects")}();if(e!==this.v){if(f>100)t();this.v=e;this.i++;l++;u++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{n()}}}});function h(t){return new c(t)}function a(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function p(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function d(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function v(t){c.call(this,void 0);this.x=t;this.s=void 0;this.g=l-1;this.f=4}(v.prototype=new c).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===l)return!0;this.g=l;this.f|=1;if(this.i>0&&!a(this)){this.f&=-2;return!0}const t=i;try{p(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;d(this);this.f&=-2;return!0};v.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}c.prototype.S.call(this,t)};v.prototype.U=function(t){if(void 0!==this.t){c.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};v.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};v.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(v.prototype,"value",{get(){if(1&this.f)t();const n=s(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function y(t){return new v(t)}function m(t){const e=t.u;t.u=void 0;if("function"==typeof e){u++;const _=i;i=void 0;try{e()}catch(n){t.f&=-2;t.f|=8;g(t);throw n}finally{i=_;n()}}}function g(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;m(t)}function b(t){if(i!==this)throw new Error("Out-of-order effect");d(this);i=t;this.f&=-2;if(8&this.f)g(this);n()}function k(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}k.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};k.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;m(this);p(this);u++;const n=i;i=this;return b.bind(this,n)};k.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=_;_=this}};k.prototype.d=function(){this.f|=8;if(!(1&this.f))g(this)};function S(t){const n=new k(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,w,C,E,U,H,N,P,$,D={},T=[],V=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,A=Array.isArray;function F(t,n){for(var e in n)t[e]=n[e];return t}function M(t){var n=t.parentNode;n&&n.removeChild(t)}function W(t,n,e){var i,_,o,r={};for(o in n)"key"==o?i=n[o]:"ref"==o?_=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,i,_,null)}function O(t,n,e,i,_){var o={type:t,props:n,key:e,ref:i,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,__h:null,constructor:void 0,__v:null==_?++C:_};return null==_&&null!=w.vnode&&w.vnode(o),o}function L(){return{current:null}}function R(t){return t.children}function I(t,n){this.props=t,this.context=n}function j(t,n){if(null==n)return t.__?j(t.__,t.__.__k.indexOf(t)+1):null;for(var e;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e)return e.__e;return"function"==typeof t.type?j(t):null}function B(t){var n,e;if(null!=(t=t.__)&&null!=t.__c){for(t.__e=t.__c.base=null,n=0;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e){t.__e=t.__c.base=e.__e;break}return B(t)}}function q(t){(!t.__d&&(t.__d=!0)&&U.push(t)&&!G.__r++||H!==w.debounceRendering)&&((H=w.debounceRendering)||N)(G)}function G(){var t,n,e,i,_,o,r,u,f;for(U.sort(P);t=U.shift();)t.__d&&(n=U.length,i=void 0,_=void 0,o=void 0,u=(r=(e=t).__v).__e,(f=e.__P)&&(i=[],_=[],(o=F({},r)).__v=r.__v+1,it(f,r,o,e.__n,void 0!==f.ownerSVGElement,null!=r.__h?[u]:null,i,null==u?j(r):u,r.__h,_),_t(i,r,_),r.__e!=u&&B(r)),U.length>n&&U.sort(P));G.__r=0}function z(t,n,e,i,_,o,r,u,f,l,s){var c,h,a,p,d,v,y,m,g,b,k=0,S=i&&i.__k||T,x=S.length,w=x,C=n.length;for(e.__k=[],c=0;c<C;c++)null!=(p=e.__k[c]=null==(p=n[c])||"boolean"==typeof p||"function"==typeof p?null:"string"==typeof p||"number"==typeof p||"bigint"==typeof p?O(null,p,null,null,p):A(p)?O(R,{children:p},null,null,null):p.__b>0?O(p.type,p.props,p.key,p.ref?p.ref:null,p.__v):p)&&(p.__=e,p.__b=e.__b+1,-1===(m=X(p,S,y=c+k,w))?a=D:(a=S[m]||D,S[m]=void 0,w--),it(t,p,a,_,o,r,u,f,l,s),d=p.__e,(h=p.ref)&&a.ref!=h&&(a.ref&&rt(a.ref,null,p),s.push(h,p.__c||d,p)),null!=d&&(null==v&&(v=d),b=!(g=a===D||null===a.__v)&&m===y,g?-1==m&&k--:m!==y&&(m===y+1?(k++,b=!0):m>y?w>C-y?(k+=m-y,b=!0):k--:k=m<y&&m==y-1?m-y:0),y=c+k,b=b||m==c&&!g,"function"!=typeof p.type||m===y&&a.__k!==p.__k?"function"==typeof p.type||b?void 0!==p.__d?(f=p.__d,p.__d=void 0):f=d.nextSibling:f=Q(t,d,f):f=J(p,f,t),"function"==typeof e.type&&(e.__d=f)));for(e.__e=v,c=x;c--;)null!=S[c]&&("function"==typeof e.type&&null!=S[c].__e&&S[c].__e==e.__d&&(e.__d=S[c].__e.nextSibling),ut(S[c],S[c]))}function J(t,n,e){for(var i,_=t.__k,o=0;_&&o<_.length;o++)(i=_[o])&&(i.__=t,n="function"==typeof i.type?J(i,n,e):Q(e,i.__e,n));return n}function K(t,n){return n=n||[],null==t||"boolean"==typeof t||(A(t)?t.some((function(t){K(t,n)})):n.push(t)),n}function Q(t,n,e){return null==e||e.parentNode!==t?t.insertBefore(n,null):n==e&&null!=n.parentNode||t.insertBefore(n,e),n.nextSibling}function X(t,n,e,i){var _=t.key,o=t.type,r=e-1,u=e+1,f=n[e];if(null===f||f&&_==f.key&&o===f.type)return e;if(i>(null!=f?1:0))for(;r>=0||u<n.length;){if(r>=0){if((f=n[r])&&_==f.key&&o===f.type)return r;r--}if(u<n.length){if((f=n[u])&&_==f.key&&o===f.type)return u;u++}}return-1}function Y(t,n,e,i,_){var o;for(o in e)"children"===o||"key"===o||o in n||tt(t,o,null,e[o],i);for(o in n)_&&"function"!=typeof n[o]||"children"===o||"key"===o||"value"===o||"checked"===o||e[o]===n[o]||tt(t,o,n[o],e[o],i)}function Z(t,n,e){"-"===n[0]?t.setProperty(n,null==e?"":e):t[n]=null==e?"":"number"!=typeof e||V.test(n)?e:e+"px"}function tt(t,n,e,i,_){var o;t:if("style"===n)if("string"==typeof e)t.style.cssText=e;else{if("string"==typeof i&&(t.style.cssText=i=""),i)for(n in i)e&&n in e||Z(t.style,n,"");if(e)for(n in e)i&&e[n]===i[n]||Z(t.style,n,e[n])}else if("o"===n[0]&&"n"===n[1])o=n!==(n=n.replace(/Capture$/,"")),n=n.toLowerCase()in t?n.toLowerCase().slice(2):n.slice(2),t.l||(t.l={}),t.l[n+o]=e,e?i||t.addEventListener(n,o?et:nt,o):t.removeEventListener(n,o?et:nt,o);else if("dangerouslySetInnerHTML"!==n){if(_)n=n.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if("width"!==n&&"height"!==n&&"href"!==n&&"list"!==n&&"form"!==n&&"tabIndex"!==n&&"download"!==n&&"rowSpan"!==n&&"colSpan"!==n&&n in t)try{t[n]=null==e?"":e;break t}catch(t){}"function"==typeof e||(null==e||!1===e&&"-"!==n[4]?t.removeAttribute(n):t.setAttribute(n,e))}}function nt(t){return this.l[t.type+!1](w.event?w.event(t):t)}function et(t){return this.l[t.type+!0](w.event?w.event(t):t)}function it(t,n,e,i,_,o,r,u,f,l){var s,c,h,a,p,d,v,y,m,g,b,k,S,x,C,E=n.type;if(void 0!==n.constructor)return null;null!=e.__h&&(f=e.__h,u=n.__e=e.__e,n.__h=null,o=[u]),(s=w.__b)&&s(n);try{t:if("function"==typeof E){if(y=n.props,m=(s=E.contextType)&&i[s.__c],g=s?m?m.props.value:s.__:i,e.__c?v=(c=n.__c=e.__c).__=c.__E:("prototype"in E&&E.prototype.render?n.__c=c=new E(y,g):(n.__c=c=new I(y,g),c.constructor=E,c.render=ft),m&&m.sub(c),c.props=y,c.state||(c.state={}),c.context=g,c.__n=i,h=c.__d=!0,c.__h=[],c._sb=[]),null==c.__s&&(c.__s=c.state),null!=E.getDerivedStateFromProps&&(c.__s==c.state&&(c.__s=F({},c.__s)),F(c.__s,E.getDerivedStateFromProps(y,c.__s))),a=c.props,p=c.state,c.__v=n,h)null==E.getDerivedStateFromProps&&null!=c.componentWillMount&&c.componentWillMount(),null!=c.componentDidMount&&c.__h.push(c.componentDidMount);else{if(null==E.getDerivedStateFromProps&&y!==a&&null!=c.componentWillReceiveProps&&c.componentWillReceiveProps(y,g),!c.__e&&(null!=c.shouldComponentUpdate&&!1===c.shouldComponentUpdate(y,c.__s,g)||n.__v===e.__v)){for(n.__v!==e.__v&&(c.props=y,c.state=c.__s,c.__d=!1),n.__e=e.__e,n.__k=e.__k,n.__k.forEach((function(t){t&&(t.__=n)})),b=0;b<c._sb.length;b++)c.__h.push(c._sb[b]);c._sb=[],c.__h.length&&r.push(c);break t}null!=c.componentWillUpdate&&c.componentWillUpdate(y,c.__s,g),null!=c.componentDidUpdate&&c.__h.push((function(){c.componentDidUpdate(a,p,d)}))}if(c.context=g,c.props=y,c.__P=t,c.__e=!1,k=w.__r,S=0,"prototype"in E&&E.prototype.render){for(c.state=c.__s,c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),x=0;x<c._sb.length;x++)c.__h.push(c._sb[x]);c._sb=[]}else do{c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),c.state=c.__s}while(c.__d&&++S<25);c.state=c.__s,null!=c.getChildContext&&(i=F(F({},i),c.getChildContext())),h||null==c.getSnapshotBeforeUpdate||(d=c.getSnapshotBeforeUpdate(a,p)),z(t,A(C=null!=s&&s.type===R&&null==s.key?s.props.children:s)?C:[C],n,e,i,_,o,r,u,f,l),c.base=n.__e,n.__h=null,c.__h.length&&r.push(c),v&&(c.__E=c.__=null)}else null==o&&n.__v===e.__v?(n.__k=e.__k,n.__e=e.__e):n.__e=ot(e.__e,n,e,i,_,o,r,f,l);(s=w.diffed)&&s(n)}catch(t){n.__v=null,(f||null!=o)&&(n.__e=u,n.__h=!!f,o[o.indexOf(u)]=null),w.__e(t,n,e)}}function _t(t,n,e){for(var i=0;i<e.length;i++)rt(e[i],e[++i],e[++i]);w.__c&&w.__c(n,t),t.some((function(n){try{t=n.__h,n.__h=[],t.some((function(t){t.call(n)}))}catch(t){w.__e(t,n.__v)}}))}function ot(t,n,e,i,_,o,r,u,f){var l,s,c,h=e.props,a=n.props,p=n.type,d=0;if("svg"===p&&(_=!0),null!=o)for(;d<o.length;d++)if((l=o[d])&&"setAttribute"in l==!!p&&(p?l.localName===p:3===l.nodeType)){t=l,o[d]=null;break}if(null==t){if(null===p)return document.createTextNode(a);t=_?document.createElementNS("http://www.w3.org/2000/svg",p):document.createElement(p,a.is&&a),o=null,u=!1}if(null===p)h===a||u&&t.data===a||(t.data=a);else{if(o=o&&x.call(t.childNodes),s=(h=e.props||D).dangerouslySetInnerHTML,c=a.dangerouslySetInnerHTML,!u){if(null!=o)for(h={},d=0;d<t.attributes.length;d++)h[t.attributes[d].name]=t.attributes[d].value;(c||s)&&(c&&(s&&c.__html==s.__html||c.__html===t.innerHTML)||(t.innerHTML=c&&c.__html||""))}if(Y(t,a,h,_,u),c)n.__k=[];else if(z(t,A(d=n.props.children)?d:[d],n,e,i,_&&"foreignObject"!==p,o,r,o?o[0]:e.__k&&j(e,0),u,f),null!=o)for(d=o.length;d--;)null!=o[d]&&M(o[d]);u||("value"in a&&void 0!==(d=a.value)&&(d!==t.value||"progress"===p&&!d||"option"===p&&d!==h.value)&&tt(t,"value",d,h.value,!1),"checked"in a&&void 0!==(d=a.checked)&&d!==t.checked&&tt(t,"checked",d,h.checked,!1))}return t}function rt(t,n,e){try{"function"==typeof t?t(n):t.current=n}catch(t){w.__e(t,e)}}function ut(t,n,e){var i,_;if(w.unmount&&w.unmount(t),(i=t.ref)&&(i.current&&i.current!==t.__e||rt(i,null,n)),null!=(i=t.__c)){if(i.componentWillUnmount)try{i.componentWillUnmount()}catch(t){w.__e(t,n)}i.base=i.__P=null,t.__c=void 0}if(i=t.__k)for(_=0;_<i.length;_++)i[_]&&ut(i[_],n,e||"function"!=typeof t.type);e||null==t.__e||M(t.__e),t.__=t.__e=t.__d=void 0}function ft(t,n,e){return this.constructor(t,e)}function lt(t,n,e){var i,_,o,r;w.__&&w.__(t,n),_=(i="function"==typeof e)?null:e&&e.__k||n.__k,o=[],r=[],it(n,t=(!i&&e||n).__k=W(R,null,[t]),_||D,D,void 0!==n.ownerSVGElement,!i&&e?[e]:_?null:n.firstChild?x.call(n.childNodes):null,o,!i&&e?e:_?_.__e:n.firstChild,i,r),_t(o,t,r)}function st(t,n){lt(t,n,st)}function ct(t,n,e){var i,_,o,r,u=F({},t.props);for(o in t.type&&t.type.defaultProps&&(r=t.type.defaultProps),n)"key"==o?i=n[o]:"ref"==o?_=n[o]:u[o]=void 0===n[o]&&void 0!==r?r[o]:n[o];return arguments.length>2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,i||t.key,_||t.ref,null)}function ht(t,n){var e={__c:n="__cC"+$++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,i;return this.getChildContext||(e=[],(i={})[n]=this,this.getChildContext=function(){return i},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,q(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=T.slice,w={__e:function(t,n,e,i){for(var _,o,r;n=n.__;)if((_=n.__c)&&!_.__)try{if((o=_.constructor)&&null!=o.getDerivedStateFromError&&(_.setState(o.getDerivedStateFromError(t)),r=_.__d),null!=_.componentDidCatch&&(_.componentDidCatch(t,i||{}),r=_.__d),r)return _.__E=_}catch(n){t=n}throw t}},C=0,E=function(t){return null!=t&&void 0===t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=F({},this.state),"function"==typeof t&&(t=t(F({},e),this.props)),t&&F(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),q(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),q(this))},I.prototype.render=R,U=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,P=function(t,n){return t.__v.__b-n.__v.__b},G.__r=0,$=0;var at,pt,dt,vt,yt=0,mt=[],gt=[],bt=w.__b,kt=w.__r,St=w.diffed,xt=w.__c,wt=w.unmount;function Ct(t,n){w.__h&&w.__h(pt,t,yt||n),yt=0;var e=pt.__H||(pt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:gt}),e.__[t]}function Et(t){return yt=1,Ut(Bt,t)}function Ut(t,n,e){var i=Ct(at++,2);if(i.t=t,!i.__c&&(i.__=[e?e(n):Bt(void 0,n),function(t){var n=i.__N?i.__N[0]:i.__[0],e=i.t(n,t);n!==e&&(i.__N=[e,i.__[1]],i.__c.setState({}))}],i.__c=pt,!pt.u)){var _=function(t,n,e){if(!i.__c.__H)return!0;var _=i.__c.__H.__.filter((function(t){return t.__c}));if(_.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return _.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&i.__c.props===t)&&(!o||o.call(this,t,n,e))};pt.u=!0;var o=pt.shouldComponentUpdate,r=pt.componentWillUpdate;pt.componentWillUpdate=function(t,n,e){if(this.__e){var i=o;o=void 0,_(t,n,e),o=i}r&&r.call(this,t,n,e)},pt.shouldComponentUpdate=_}return i.__N||i.__}function Ht(t,n){var e=Ct(at++,3);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__H.__h.push(e))}function Nt(t,n){var e=Ct(at++,4);!w.__s&&jt(e.__H,n)&&(e.__=t,e.i=n,pt.__h.push(e))}function Pt(t){return yt=5,Dt((function(){return{current:t}}),[])}function $t(t,n,e){yt=6,Nt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function Dt(t,n){var e=Ct(at++,7);return jt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Tt(t,n){return yt=8,Dt((function(){return t}),n)}function Vt(t){var n=pt.context[t.__c],e=Ct(at++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(pt)),n.props.value):t.__}function At(t,n){w.useDebugValue&&w.useDebugValue(n?n(t):t)}function Ft(t){var n=Ct(at++,10),e=Et();return n.__=t,pt.componentDidCatch||(pt.componentDidCatch=function(t,i){n.__&&n.__(t,i),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Mt(){var t=Ct(at++,11);if(!t.__){for(var n=pt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Wt(){for(var t;t=mt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(Rt),t.__H.__h.forEach(It),t.__H.__h=[]}catch(u){t.__H.__h=[],w.__e(u,t.__v)}}w.__b=function(t){pt=null,bt&&bt(t)},w.__r=function(t){kt&&kt(t),at=0;var n=(pt=t.__c).__H;n&&(dt===pt?(n.__h=[],pt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=gt,t.__N=t.i=void 0}))):(n.__h.forEach(Rt),n.__h.forEach(It),n.__h=[],at=0)),dt=pt},w.diffed=function(t){St&&St(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==mt.push(n)&&vt===w.requestAnimationFrame||((vt=w.requestAnimationFrame)||Lt)(Wt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==gt&&(t.__=t.__V),t.i=void 0,t.__V=gt}))),dt=pt=null},w.__c=function(t,n){n.some((function(t){try{t.__h.forEach(Rt),t.__h=t.__h.filter((function(t){return!t.__||It(t)}))}catch(s){n.some((function(t){t.__h&&(t.__h=[])})),n=[],w.__e(s,t.__v)}})),xt&&xt(t,n)},w.unmount=function(t){wt&&wt(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{Rt(t)}catch(t){n=t}})),e.__H=void 0,n&&w.__e(n,e.__v))};var Ot="function"==typeof requestAnimationFrame;function Lt(t){var n,e=function(){clearTimeout(i),Ot&&cancelAnimationFrame(n),setTimeout(t)},i=setTimeout(e,100);Ot&&(n=requestAnimationFrame(e))}function Rt(t){var n=pt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),pt=n}function It(t){var n=pt;t.__c=t.__(),pt=n}function jt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function Bt(t,n){return"function"==typeof n?n(t):n}function qt(t,n){w[t]=n.bind(null,w[t]||(()=>{}))}let Gt,zt;function Jt(t){if(zt)zt();zt=t&&t.S()}function Kt({data:t}){const n=Xt(t);n.value=t;const e=Dt(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!E(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return y(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Kt.displayName="_st";Object.defineProperties(c.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Kt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});qt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let i in e){if("children"===i)continue;let _=e[i];if(_ instanceof c){if(!t)n.__np=t={};t[i]=_;e[i]=_.peek()}}}t(n)});qt("__r",(t,n)=>{Jt();let e,i=n.__c;if(i){i.__$f&=-2;e=i.__$u;if(void 0===e)i.__$u=e=function(t){let n;S((function(){n=this}));n.c=()=>{i.__$f|=1;i.setState({})};return n}()}Gt=i;Jt(e);t(n)});qt("__e",(t,n,e,i)=>{Jt();Gt=void 0;t(n,e,i)});qt("diffed",(t,n)=>{Jt();Gt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,i=n.props;if(t){let n=e.U;if(n)for(let e in n){let i=n[e];if(void 0!==i&&!(e in t)){i.d();n[e]=void 0}}else{n={};e.U=n}for(let _ in t){let o=n[_],r=t[_];if(void 0===o){o=Qt(e,_,r,i);n[_]=o}else o.o(r,i)}}}t(n)});function Qt(t,n,e,i){const _=n in t&&void 0===t.ownerSVGElement,o=h(e);return{o:(t,n)=>{o.value=t;i=n},d:S(()=>{const e=o.value.value;if(i[n]!==e){i[n]=e;if(_)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}qt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});qt("__h",(t,n,e,i)=>{if(i<3||9===i)n.__$f|=2;t(n,e,i)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let i in n)return!0;for(let i in t)if("__source"!==i&&t[i]!==this.props[i])return!0;for(let i in this.props)if(!(i in t))return!0;return!1};function Xt(t){return Dt(()=>h(t),[])}function Yt(t){const n=Pt(t);n.current=t;Gt.__$f|=4;return Dt(()=>y(()=>n.current()),[])}function Zt(t){const n=Pt(t);n.current=t;Ht(()=>S(()=>n.current()),[])}var tn=function(t,n,e,i){var _;n[0]=0;for(var o=1;o<n.length;o++){var r=n[o++],u=n[o]?(n[0]|=r?1:2,e[n[o++]]):n[++o];3===r?i[0]=u:4===r?i[1]=Object.assign(i[1]||{},u):5===r?(i[1]=i[1]||{})[n[++o]]=u:6===r?i[1][n[++o]]+=u+"":r?(_=t.apply(u,tn(t,u,e,["",null])),i.push(_),u[0]?n[0]|=2:(n[o-2]=0,n[o]=_)):i.push(u)}return i},nn=new Map;function en(t){var n=nn.get(this);return n||(n=new Map,nn.set(this,n)),(n=tn(this,n.get(t)||(n.set(t,n=function(t){for(var n,e,i=1,_="",o="",r=[0],u=function(t){1===i&&(t||(_=_.replace(/^\s*\n\s*|\s*\n\s*$/g,"")))?r.push(0,t,_):3===i&&(t||_)?(r.push(3,t,_),i=2):2===i&&"..."===_&&t?r.push(4,t,0):2===i&&_&&!t?r.push(5,0,!0,_):i>=5&&((_||!t&&5===i)&&(r.push(i,0,_,e),i=6),t&&(r.push(i,t,0,e),i=6)),_=""},f=0;f<t.length;f++){f&&(1===i&&u(),u(f));for(var l=0;l<t[f].length;l++)n=t[f][l],1===i?"<"===n?(u(),r=[r],i=3):_+=n:4===i?"--"===_&&">"===n?(i=1,_=""):_=n+_[0]:o?n===o?o="":_+=n:'"'===n||"'"===n?o=n:">"===n?(u(),i=1):i&&("="===n?(i=5,e=_,_=""):"/"===n&&(i<5||">"===t[f][l+1])?(u(),3===i&&(r=r[0]),i=r,(r=r[0]).push(2,0,i),i=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),i=2):_+=n),3===i&&"!--"===_&&(i=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var _n=en.bind(W);export{I as Component,R as Fragment,c as Signal,e as batch,ct as cloneElement,y as computed,ht as createContext,W as createElement,L as createRef,S as effect,W as h,_n as html,st as hydrate,E as isValidElement,w as options,lt as render,h as signal,K as toChildArray,r as untracked,Tt as useCallback,Yt as useComputed,Vt as useContext,At as useDebugValue,Ht as useEffect,Ft as useErrorBoundary,Mt as useId,$t as useImperativeHandle,Nt as useLayoutEffect,Dt as useMemo,Ut as useReducer,Pt as useRef,Xt as useSignal,Zt as useSignalEffect,Et as useState};+function t(){throw new Error("Cycle detected")}const n=Symbol.for("preact-signals");function e(){if(f>1){f--;return}let t,n=!1;while(void 0!==o){let _=o;o=void 0;l++;while(void 0!==_){const i=_.o;_.o=void 0;_.f&=-3;if(!(8&_.f)&&p(_))try{_.c()}catch(e){if(!n){t=e;n=!0}}_=i}}l=0;f--;if(n)throw t}function _(t){if(f>0)return t();f++;try{return t()}finally{e()}}let i,o,r=0;function u(t){if(r>0)return t();const n=i;i=void 0;r++;try{return t()}finally{r--;i=n}}let f=0,l=0,s=0;function c(t){if(void 0===i)return;let n=t.n;if(void 0===n||n.t!==i){n={i:0,S:t,p:i.s,n:void 0,t:i,e:void 0,x:void 0,r:n};if(void 0!==i.s)i.s.n=n;i.s=n;t.n=n;if(32&i.f)t.S(n);return n}else if(-1===n.i){n.i=0;if(void 0!==n.n){n.n.p=n.p;if(void 0!==n.p)n.p.n=n.n;n.p=i.s;n.n=void 0;i.s.n=n;i.s=n}return n}}function h(t){this.v=t;this.i=0;this.n=void 0;this.t=void 0}h.prototype.brand=n;h.prototype.h=function(){return!0};h.prototype.S=function(t){if(this.t!==t&&void 0===t.e){t.x=this.t;if(void 0!==this.t)this.t.e=t;this.t=t}};h.prototype.U=function(t){if(void 0!==this.t){const n=t.e,e=t.x;if(void 0!==n){n.x=e;t.e=void 0}if(void 0!==e){e.e=n;t.x=void 0}if(t===this.t)this.t=e}};h.prototype.subscribe=function(t){const n=this;return w((function(){const e=n.value,_=32&this.f;this.f&=-33;try{t(e)}finally{this.f|=_}}))};h.prototype.valueOf=function(){return this.value};h.prototype.toString=function(){return this.value+""};h.prototype.toJSON=function(){return this.value};h.prototype.peek=function(){return this.v};Object.defineProperty(h.prototype,"value",{get(){const t=c(this);if(void 0!==t)t.i=this.i;return this.v},set(n){if(i instanceof y)!function(){throw new Error("Computed cannot have side-effects")}();if(n!==this.v){if(l>100)t();this.v=n;this.i++;s++;f++;try{for(let t=this.t;void 0!==t;t=t.x)t.t.N()}finally{e()}}}});function a(t){return new h(t)}function p(t){for(let n=t.s;void 0!==n;n=n.n)if(n.S.i!==n.i||!n.S.h()||n.S.i!==n.i)return!0;return!1}function d(t){for(let n=t.s;void 0!==n;n=n.n){const e=n.S.n;if(void 0!==e)n.r=e;n.S.n=n;n.i=-1;if(void 0===n.n){t.s=n;break}}}function v(t){let n,e=t.s;while(void 0!==e){const t=e.p;if(-1===e.i){e.S.U(e);if(void 0!==t)t.n=e.n;if(void 0!==e.n)e.n.p=t}else n=e;e.S.n=e.r;if(void 0!==e.r)e.r=void 0;e=t}t.s=n}function y(t){h.call(this,void 0);this.x=t;this.s=void 0;this.g=s-1;this.f=4}(y.prototype=new h).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===s)return!0;this.g=s;this.f|=1;if(this.i>0&&!p(this)){this.f&=-2;return!0}const t=i;try{d(this);i=this;const t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(t){this.v=t;this.f|=16;this.i++}i=t;v(this);this.f&=-2;return!0};y.prototype.S=function(t){if(void 0===this.t){this.f|=36;for(let t=this.s;void 0!==t;t=t.n)t.S.S(t)}h.prototype.S.call(this,t)};y.prototype.U=function(t){if(void 0!==this.t){h.prototype.U.call(this,t);if(void 0===this.t){this.f&=-33;for(let t=this.s;void 0!==t;t=t.n)t.S.U(t)}}};y.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(let t=this.t;void 0!==t;t=t.x)t.t.N()}};y.prototype.peek=function(){if(!this.h())t();if(16&this.f)throw this.v;return this.v};Object.defineProperty(y.prototype,"value",{get(){if(1&this.f)t();const n=c(this);this.h();if(void 0!==n)n.i=this.i;if(16&this.f)throw this.v;return this.v}});function m(t){return new y(t)}function g(t){const n=t.u;t.u=void 0;if("function"==typeof n){f++;const _=i;i=void 0;try{n()}catch(n){t.f&=-2;t.f|=8;b(t);throw n}finally{i=_;e()}}}function b(t){for(let n=t.s;void 0!==n;n=n.n)n.S.U(n);t.x=void 0;t.s=void 0;g(t)}function k(t){if(i!==this)throw new Error("Out-of-order effect");v(this);i=t;this.f&=-2;if(8&this.f)b(this);e()}function S(t){this.x=t;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}S.prototype.c=function(){const t=this.S();try{if(8&this.f)return;if(void 0===this.x)return;const n=this.x();if("function"==typeof n)this.u=n}finally{t()}};S.prototype.S=function(){if(1&this.f)t();this.f|=1;this.f&=-9;g(this);d(this);f++;const n=i;i=this;return k.bind(this,n)};S.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=o;o=this}};S.prototype.d=function(){this.f|=8;if(!(1&this.f))b(this)};function w(t){const n=new S(t);try{n.c()}catch(t){n.d();throw t}return n.d.bind(n)}var x,C,E,U,H,P,N,$,D,T={},V=[],A=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,F=Array.isArray;function M(t,n){for(var e in n)t[e]=n[e];return t}function W(t){var n=t.parentNode;n&&n.removeChild(t)}function L(t,n,e){var _,i,o,r={};for(o in n)"key"==o?_=n[o]:"ref"==o?i=n[o]:r[o]=n[o];if(arguments.length>2&&(r.children=arguments.length>3?x.call(arguments,2):e),"function"==typeof t&&null!=t.defaultProps)for(o in t.defaultProps)void 0===r[o]&&(r[o]=t.defaultProps[o]);return O(t,r,_,i,null)}function O(t,n,e,_,i){var o={type:t,props:n,key:e,ref:_,__k:null,__:null,__b:0,__e:null,__d:void 0,__c:null,constructor:void 0,__v:null==i?++E:i,__i:-1,__u:0};return null==i&&null!=C.vnode&&C.vnode(o),o}function R(){return{current:null}}function j(t){return t.children}function I(t,n){this.props=t,this.context=n}function q(t,n){if(null==n)return t.__?q(t.__,t.__i+1):null;for(var e;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e)return e.__e;return"function"==typeof t.type?q(t):null}function B(t,n,e){var _,i=t.__v,o=i.__e,r=t.__P;if(r)return(_=M({},i)).__v=i.__v+1,C.vnode&&C.vnode(_),it(r,_,i,t.__n,void 0!==r.ownerSVGElement,32&i.__u?[o]:null,n,null==o?q(i):o,!!(32&i.__u),e),_.__v=i.__v,_.__.__k[_.__i]=_,_.__d=void 0,_.__e!=o&&G(_),_}function G(t){var n,e;if(null!=(t=t.__)&&null!=t.__c){for(t.__e=t.__c.base=null,n=0;n<t.__k.length;n++)if(null!=(e=t.__k[n])&&null!=e.__e){t.__e=t.__c.base=e.__e;break}return G(t)}}function z(t){(!t.__d&&(t.__d=!0)&&H.push(t)&&!J.__r++||P!==C.debounceRendering)&&((P=C.debounceRendering)||N)(J)}function J(){var t,n,e,_=[],i=[];for(H.sort($);t=H.shift();)t.__d&&(e=H.length,n=B(t,_,i)||n,0===e||H.length>e?(ot(_,n,i),i.length=_.length=0,n=void 0,H.sort($)):n&&C.__c&&C.__c(n,V));n&&ot(_,n,i),J.__r=0}function K(t,n,e,_,i,o,r,u,f,l,s){var c,h,a,p,d,v=_&&_.__k||V,y=n.length;for(e.__d=f,Q(e,n,v),f=e.__d,c=0;c<y;c++)null!=(a=e.__k[c])&&"boolean"!=typeof a&&"function"!=typeof a&&(h=-1===a.__i?T:v[a.__i]||T,a.__i=c,it(t,a,h,i,o,r,u,f,l,s),p=a.__e,a.ref&&h.ref!=a.ref&&(h.ref&&ut(h.ref,null,a),s.push(a.ref,a.__c||p,a)),null==d&&null!=p&&(d=p),65536&a.__u||h.__k===a.__k?f=X(a,f,t):"function"==typeof a.type&&void 0!==a.__d?f=a.__d:p&&(f=p.nextSibling),a.__d=void 0,a.__u&=-196609);e.__d=f,e.__e=d}function Q(t,n,e){var _,i,o,r,u,f=n.length,l=e.length,s=l,c=0;for(t.__k=[],_=0;_<f;_++)r=_+c,null!=(i=t.__k[_]=null==(i=n[_])||"boolean"==typeof i||"function"==typeof i?null:"string"==typeof i||"number"==typeof i||"bigint"==typeof i||i.constructor==String?O(null,i,null,null,null):F(i)?O(j,{children:i},null,null,null):void 0===i.constructor&&i.__b>0?O(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):i)?(i.__=t,i.__b=t.__b+1,u=Z(i,e,r,s),i.__i=u,o=null,-1!==u&&(s--,(o=e[u])&&(o.__u|=131072)),null==o||null===o.__v?(-1==u&&c--,"function"!=typeof i.type&&(i.__u|=65536)):u!==r&&(u===r+1?c++:u>r?s>f-r?c+=u-r:c--:u<r?u==r-1&&(c=u-r):c=0,u!==_+c&&(i.__u|=65536))):(o=e[r])&&null==o.key&&o.__e&&0==(131072&o.__u)&&(o.__e==t.__d&&(t.__d=q(o)),ft(o,o,!1),e[r]=null,s--);if(s)for(_=0;_<l;_++)null!=(o=e[_])&&0==(131072&o.__u)&&(o.__e==t.__d&&(t.__d=q(o)),ft(o,o))}function X(t,n,e){var _,i;if("function"==typeof t.type){for(_=t.__k,i=0;_&&i<_.length;i++)_[i]&&(_[i].__=t,n=X(_[i],n,e));return n}t.__e!=n&&(e.insertBefore(t.__e,n||null),n=t.__e);do{n=n&&n.nextSibling}while(null!=n&&8===n.nodeType);return n}function Y(t,n){return n=n||[],null==t||"boolean"==typeof t||(F(t)?t.some((function(t){Y(t,n)})):n.push(t)),n}function Z(t,n,e,_){var i=t.key,o=t.type,r=e-1,u=e+1,f=n[e];if(null===f||f&&i==f.key&&o===f.type&&0==(131072&f.__u))return e;if(_>(null!=f&&0==(131072&f.__u)?1:0))for(;r>=0||u<n.length;){if(r>=0){if((f=n[r])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return r;r--}if(u<n.length){if((f=n[u])&&0==(131072&f.__u)&&i==f.key&&o===f.type)return u;u++}}return-1}function tt(t,n,e){"-"===n[0]?t.setProperty(n,null==e?"":e):t[n]=null==e?"":"number"!=typeof e||A.test(n)?e:e+"px"}function nt(t,n,e,_,i){var o;t:if("style"===n)if("string"==typeof e)t.style.cssText=e;else{if("string"==typeof _&&(t.style.cssText=_=""),_)for(n in _)e&&n in e||tt(t.style,n,"");if(e)for(n in e)_&&e[n]===_[n]||tt(t.style,n,e[n])}else if("o"===n[0]&&"n"===n[1])o=n!==(n=n.replace(/(PointerCapture)$|Capture$/i,"$1")),n=n.toLowerCase()in t?n.toLowerCase().slice(2):n.slice(2),t.l||(t.l={}),t.l[n+o]=e,e?_?e.u=_.u:(e.u=Date.now(),t.addEventListener(n,o?_t:et,o)):t.removeEventListener(n,o?_t:et,o);else{if(i)n=n.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if("width"!==n&&"height"!==n&&"href"!==n&&"list"!==n&&"form"!==n&&"tabIndex"!==n&&"download"!==n&&"rowSpan"!==n&&"colSpan"!==n&&"role"!==n&&n in t)try{t[n]=null==e?"":e;break t}catch(t){}"function"==typeof e||(null==e||!1===e&&"-"!==n[4]?t.removeAttribute(n):t.setAttribute(n,e))}}function et(t){if(this.l){var n=this.l[t.type+!1];if(t.t){if(t.t<=n.u)return}else t.t=Date.now();return n(C.event?C.event(t):t)}}function _t(t){if(this.l)return this.l[t.type+!0](C.event?C.event(t):t)}function it(t,n,e,_,i,o,r,u,f,l){var s,c,h,a,p,d,v,y,m,g,b,k,S,w,x,E=n.type;if(void 0!==n.constructor)return null;128&e.__u&&(f=!!(32&e.__u),o=[u=n.__e=e.__e]),(s=C.__b)&&s(n);t:if("function"==typeof E)try{if(y=n.props,m=(s=E.contextType)&&_[s.__c],g=s?m?m.props.value:s.__:_,e.__c?v=(c=n.__c=e.__c).__=c.__E:("prototype"in E&&E.prototype.render?n.__c=c=new E(y,g):(n.__c=c=new I(y,g),c.constructor=E,c.render=lt),m&&m.sub(c),c.props=y,c.state||(c.state={}),c.context=g,c.__n=_,h=c.__d=!0,c.__h=[],c._sb=[]),null==c.__s&&(c.__s=c.state),null!=E.getDerivedStateFromProps&&(c.__s==c.state&&(c.__s=M({},c.__s)),M(c.__s,E.getDerivedStateFromProps(y,c.__s))),a=c.props,p=c.state,c.__v=n,h)null==E.getDerivedStateFromProps&&null!=c.componentWillMount&&c.componentWillMount(),null!=c.componentDidMount&&c.__h.push(c.componentDidMount);else{if(null==E.getDerivedStateFromProps&&y!==a&&null!=c.componentWillReceiveProps&&c.componentWillReceiveProps(y,g),!c.__e&&(null!=c.shouldComponentUpdate&&!1===c.shouldComponentUpdate(y,c.__s,g)||n.__v===e.__v)){for(n.__v!==e.__v&&(c.props=y,c.state=c.__s,c.__d=!1),n.__e=e.__e,n.__k=e.__k,n.__k.forEach((function(t){t&&(t.__=n)})),b=0;b<c._sb.length;b++)c.__h.push(c._sb[b]);c._sb=[],c.__h.length&&r.push(c);break t}null!=c.componentWillUpdate&&c.componentWillUpdate(y,c.__s,g),null!=c.componentDidUpdate&&c.__h.push((function(){c.componentDidUpdate(a,p,d)}))}if(c.context=g,c.props=y,c.__P=t,c.__e=!1,k=C.__r,S=0,"prototype"in E&&E.prototype.render){for(c.state=c.__s,c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),w=0;w<c._sb.length;w++)c.__h.push(c._sb[w]);c._sb=[]}else do{c.__d=!1,k&&k(n),s=c.render(c.props,c.state,c.context),c.state=c.__s}while(c.__d&&++S<25);c.state=c.__s,null!=c.getChildContext&&(_=M(M({},_),c.getChildContext())),h||null==c.getSnapshotBeforeUpdate||(d=c.getSnapshotBeforeUpdate(a,p)),K(t,F(x=null!=s&&s.type===j&&null==s.key?s.props.children:s)?x:[x],n,e,_,i,o,r,u,f,l),c.base=n.__e,n.__u&=-161,c.__h.length&&r.push(c),v&&(c.__E=c.__=null)}catch(t){n.__v=null,f||null!=o?(n.__e=u,n.__u|=f?160:32,o[o.indexOf(u)]=null):(n.__e=e.__e,n.__k=e.__k),C.__e(t,n,e)}else null==o&&n.__v===e.__v?(n.__k=e.__k,n.__e=e.__e):n.__e=rt(e.__e,n,e,_,i,o,r,f,l);(s=C.diffed)&&s(n)}function ot(t,n,e){for(var _=0;_<e.length;_++)ut(e[_],e[++_],e[++_]);C.__c&&C.__c(n,t),t.some((function(n){try{t=n.__h,n.__h=[],t.some((function(t){t.call(n)}))}catch(t){C.__e(t,n.__v)}}))}function rt(t,n,e,_,i,o,r,u,f){var l,s,c,h,a,p,d,v=e.props,y=n.props,m=n.type;if("svg"===m&&(i=!0),null!=o)for(l=0;l<o.length;l++)if((a=o[l])&&"setAttribute"in a==!!m&&(m?a.localName===m:3===a.nodeType)){t=a,o[l]=null;break}if(null==t){if(null===m)return document.createTextNode(y);t=i?document.createElementNS("http://www.w3.org/2000/svg",m):document.createElement(m,y.is&&y),o=null,u=!1}if(null===m)v===y||u&&t.data===y||(t.data=y);else{if(o=o&&x.call(t.childNodes),v=e.props||T,!u&&null!=o)for(v={},l=0;l<t.attributes.length;l++)v[(a=t.attributes[l]).name]=a.value;for(l in v)a=v[l],"children"==l||("dangerouslySetInnerHTML"==l?c=a:"key"===l||l in y||nt(t,l,null,a,i));for(l in y)a=y[l],"children"==l?h=a:"dangerouslySetInnerHTML"==l?s=a:"value"==l?p=a:"checked"==l?d=a:"key"===l||u&&"function"!=typeof a||v[l]===a||nt(t,l,a,v[l],i);if(s)u||c&&(s.__html===c.__html||s.__html===t.innerHTML)||(t.innerHTML=s.__html),n.__k=[];else if(c&&(t.innerHTML=""),K(t,F(h)?h:[h],n,e,_,i&&"foreignObject"!==m,o,r,o?o[0]:e.__k&&q(e,0),u,f),null!=o)for(l=o.length;l--;)null!=o[l]&&W(o[l]);u||(l="value",void 0!==p&&(p!==t[l]||"progress"===m&&!p||"option"===m&&p!==v[l])&&nt(t,l,p,v[l],!1),l="checked",void 0!==d&&d!==t[l]&&nt(t,l,d,v[l],!1))}return t}function ut(t,n,e){try{"function"==typeof t?t(n):t.current=n}catch(t){C.__e(t,e)}}function ft(t,n,e){var _,i;if(C.unmount&&C.unmount(t),(_=t.ref)&&(_.current&&_.current!==t.__e||ut(_,null,n)),null!=(_=t.__c)){if(_.componentWillUnmount)try{_.componentWillUnmount()}catch(t){C.__e(t,n)}_.base=_.__P=null,t.__c=void 0}if(_=t.__k)for(i=0;i<_.length;i++)_[i]&&ft(_[i],n,e||"function"!=typeof t.type);e||null==t.__e||W(t.__e),t.__=t.__e=t.__d=void 0}function lt(t,n,e){return this.constructor(t,e)}function st(t,n,e){var _,i,o,r;C.__&&C.__(t,n),i=(_="function"==typeof e)?null:e&&e.__k||n.__k,o=[],r=[],it(n,t=(!_&&e||n).__k=L(j,null,[t]),i||T,T,void 0!==n.ownerSVGElement,!_&&e?[e]:i?null:n.firstChild?x.call(n.childNodes):null,o,!_&&e?e:i?i.__e:n.firstChild,_,r),t.__d=void 0,ot(o,t,r)}function ct(t,n){st(t,n,ct)}function ht(t,n,e){var _,i,o,r,u=M({},t.props);for(o in t.type&&t.type.defaultProps&&(r=t.type.defaultProps),n)"key"==o?_=n[o]:"ref"==o?i=n[o]:u[o]=void 0===n[o]&&void 0!==r?r[o]:n[o];return arguments.length>2&&(u.children=arguments.length>3?x.call(arguments,2):e),O(t.type,u,_||t.key,i||t.ref,null)}function at(t,n){var e={__c:n="__cC"+D++,__:t,Consumer:function(t,n){return t.children(n)},Provider:function(t){var e,_;return this.getChildContext||(e=[],(_={})[n]=this,this.getChildContext=function(){return _},this.shouldComponentUpdate=function(t){this.props.value!==t.value&&e.some((function(t){t.__e=!0,z(t)}))},this.sub=function(t){e.push(t);var n=t.componentWillUnmount;t.componentWillUnmount=function(){e.splice(e.indexOf(t),1),n&&n.call(t)}}),t.children}};return e.Provider.__=e.Consumer.contextType=e}x=V.slice,C={__e:function(t,n,e,_){for(var i,o,r;n=n.__;)if((i=n.__c)&&!i.__)try{if((o=i.constructor)&&null!=o.getDerivedStateFromError&&(i.setState(o.getDerivedStateFromError(t)),r=i.__d),null!=i.componentDidCatch&&(i.componentDidCatch(t,_||{}),r=i.__d),r)return i.__E=i}catch(n){t=n}throw t}},E=0,U=function(t){return null!=t&&null==t.constructor},I.prototype.setState=function(t,n){var e;e=null!=this.__s&&this.__s!==this.state?this.__s:this.__s=M({},this.state),"function"==typeof t&&(t=t(M({},e),this.props)),t&&M(e,t),null!=t&&this.__v&&(n&&this._sb.push(n),z(this))},I.prototype.forceUpdate=function(t){this.__v&&(this.__e=!0,t&&this.__h.push(t),z(this))},I.prototype.render=j,H=[],N="function"==typeof Promise?Promise.prototype.then.bind(Promise.resolve()):setTimeout,$=function(t,n){return t.__v.__b-n.__v.__b},J.__r=0,D=0;var pt,dt,vt,yt,mt=0,gt=[],bt=[],kt=C,St=kt.__b,wt=kt.__r,xt=kt.diffed,Ct=kt.__c,Et=kt.unmount,Ut=kt.__;function Ht(t,n){kt.__h&&kt.__h(dt,t,mt||n),mt=0;var e=dt.__H||(dt.__H={__:[],__h:[]});return t>=e.__.length&&e.__.push({__V:bt}),e.__[t]}function Pt(t){return mt=1,Nt(zt,t)}function Nt(t,n,e){var _=Ht(pt++,2);if(_.t=t,!_.__c&&(_.__=[e?e(n):zt(void 0,n),function(t){var n=_.__N?_.__N[0]:_.__[0],e=_.t(n,t);n!==e&&(_.__N=[e,_.__[1]],_.__c.setState({}))}],_.__c=dt,!dt.u)){var i=function(t,n,e){if(!_.__c.__H)return!0;var i=_.__c.__H.__.filter((function(t){return!!t.__c}));if(i.every((function(t){return!t.__N})))return!o||o.call(this,t,n,e);var r=!1;return i.forEach((function(t){if(t.__N){var n=t.__[0];t.__=t.__N,t.__N=void 0,n!==t.__[0]&&(r=!0)}})),!(!r&&_.__c.props===t)&&(!o||o.call(this,t,n,e))};dt.u=!0;var o=dt.shouldComponentUpdate,r=dt.componentWillUpdate;dt.componentWillUpdate=function(t,n,e){if(this.__e){var _=o;o=void 0,i(t,n,e),o=_}r&&r.call(this,t,n,e)},dt.shouldComponentUpdate=i}return _.__N||_.__}function $t(t,n){var e=Ht(pt++,3);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__H.__h.push(e))}function Dt(t,n){var e=Ht(pt++,4);!kt.__s&&Gt(e.__H,n)&&(e.__=t,e.i=n,dt.__h.push(e))}function Tt(t){return mt=5,At((function(){return{current:t}}),[])}function Vt(t,n,e){mt=6,Dt((function(){return"function"==typeof t?(t(n()),function(){return t(null)}):t?(t.current=n(),function(){return t.current=null}):void 0}),null==e?e:e.concat(t))}function At(t,n){var e=Ht(pt++,7);return Gt(e.__H,n)?(e.__V=t(),e.i=n,e.__h=t,e.__V):e.__}function Ft(t,n){return mt=8,At((function(){return t}),n)}function Mt(t){var n=dt.context[t.__c],e=Ht(pt++,9);return e.c=t,n?(null==e.__&&(e.__=!0,n.sub(dt)),n.props.value):t.__}function Wt(t,n){kt.useDebugValue&&kt.useDebugValue(n?n(t):t)}function Lt(t){var n=Ht(pt++,10),e=Pt();return n.__=t,dt.componentDidCatch||(dt.componentDidCatch=function(t,_){n.__&&n.__(t,_),e[1](t)}),[e[0],function(){e[1](void 0)}]}function Ot(){var t=Ht(pt++,11);if(!t.__){for(var n=dt.__v;null!==n&&!n.__m&&null!==n.__;)n=n.__;var e=n.__m||(n.__m=[0,0]);t.__="P"+e[0]+"-"+e[1]++}return t.__}function Rt(){for(var t;t=gt.shift();)if(t.__P&&t.__H)try{t.__H.__h.forEach(qt),t.__H.__h.forEach(Bt),t.__H.__h=[]}catch(n){t.__H.__h=[],kt.__e(n,t.__v)}}kt.__b=function(t){dt=null,St&&St(t)},kt.__=function(t,n){t&&n.__k&&n.__k.__m&&(t.__m=n.__k.__m),Ut&&Ut(t,n)},kt.__r=function(t){wt&&wt(t),pt=0;var n=(dt=t.__c).__H;n&&(vt===dt?(n.__h=[],dt.__h=[],n.__.forEach((function(t){t.__N&&(t.__=t.__N),t.__V=bt,t.__N=t.i=void 0}))):(n.__h.forEach(qt),n.__h.forEach(Bt),n.__h=[],pt=0)),vt=dt},kt.diffed=function(t){xt&&xt(t);var n=t.__c;n&&n.__H&&(n.__H.__h.length&&(1!==gt.push(n)&&yt===kt.requestAnimationFrame||((yt=kt.requestAnimationFrame)||It)(Rt)),n.__H.__.forEach((function(t){t.i&&(t.__H=t.i),t.__V!==bt&&(t.__=t.__V),t.i=void 0,t.__V=bt}))),vt=dt=null},kt.__c=function(t,n){n.some((function(t){try{t.__h.forEach(qt),t.__h=t.__h.filter((function(t){return!t.__||Bt(t)}))}catch(u){n.some((function(t){t.__h&&(t.__h=[])})),n=[],kt.__e(u,t.__v)}})),Ct&&Ct(t,n)},kt.unmount=function(t){Et&&Et(t);var n,e=t.__c;e&&e.__H&&(e.__H.__.forEach((function(t){try{qt(t)}catch(t){n=t}})),e.__H=void 0,n&&kt.__e(n,e.__v))};var jt="function"==typeof requestAnimationFrame;function It(t){var n,e=function(){clearTimeout(_),jt&&cancelAnimationFrame(n),setTimeout(t)},_=setTimeout(e,100);jt&&(n=requestAnimationFrame(e))}function qt(t){var n=dt,e=t.__c;"function"==typeof e&&(t.__c=void 0,e()),dt=n}function Bt(t){var n=dt;t.__c=t.__(),dt=n}function Gt(t,n){return!t||t.length!==n.length||n.some((function(n,e){return n!==t[e]}))}function zt(t,n){return"function"==typeof n?n(t):n}function Jt(t,n){C[t]=n.bind(null,C[t]||(()=>{}))}let Kt,Qt;function Xt(t){if(Qt)Qt();Qt=t&&t.S()}function Yt({data:t}){const n=tn(t);n.value=t;const e=At(()=>{let t=this.__v;while(t=t.__)if(t.__c){t.__c.__$f|=4;break}this.__$u.c=()=>{var t;if(!U(e.peek())&&3===(null==(t=this.base)?void 0:t.nodeType))this.base.data=e.peek();else{this.__$f|=1;this.setState({})}};return m(()=>{let t=n.value.value;return 0===t?0:!0===t?"":t||""})},[]);return e.value}Yt.displayName="_st";Object.defineProperties(h.prototype,{constructor:{configurable:!0,value:void 0},type:{configurable:!0,value:Yt},props:{configurable:!0,get(){return{data:this}}},__b:{configurable:!0,value:1}});Jt("__b",(t,n)=>{if("string"==typeof n.type){let t,e=n.props;for(let _ in e){if("children"===_)continue;let i=e[_];if(i instanceof h){if(!t)n.__np=t={};t[_]=i;e[_]=i.peek()}}}t(n)});Jt("__r",(t,n)=>{Xt();let e,_=n.__c;if(_){_.__$f&=-2;e=_.__$u;if(void 0===e)_.__$u=e=function(t){let n;w((function(){n=this}));n.c=()=>{_.__$f|=1;_.setState({})};return n}()}Kt=_;Xt(e);t(n)});Jt("__e",(t,n,e,_)=>{Xt();Kt=void 0;t(n,e,_)});Jt("diffed",(t,n)=>{Xt();Kt=void 0;let e;if("string"==typeof n.type&&(e=n.__e)){let t=n.__np,_=n.props;if(t){let n=e.U;if(n)for(let e in n){let _=n[e];if(void 0!==_&&!(e in t)){_.d();n[e]=void 0}}else{n={};e.U=n}for(let i in t){let o=n[i],r=t[i];if(void 0===o){o=Zt(e,i,r,_);n[i]=o}else o.o(r,_)}}}t(n)});function Zt(t,n,e,_){const i=n in t&&void 0===t.ownerSVGElement,o=a(e);return{o:(t,n)=>{o.value=t;_=n},d:w(()=>{const e=o.value.value;if(_[n]!==e){_[n]=e;if(i)t[n]=e;else if(e)t.setAttribute(n,e);else t.removeAttribute(n)}})}}Jt("unmount",(t,n)=>{if("string"==typeof n.type){let t=n.__e;if(t){const n=t.U;if(n){t.U=void 0;for(let t in n){let e=n[t];if(e)e.d()}}}}else{let t=n.__c;if(t){const n=t.__$u;if(n){t.__$u=void 0;n.d()}}}t(n)});Jt("__h",(t,n,e,_)=>{if(_<3||9===_)n.__$f|=2;t(n,e,_)});I.prototype.shouldComponentUpdate=function(t,n){const e=this.__$u;if(!(e&&void 0!==e.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(let _ in n)return!0;for(let _ in t)if("__source"!==_&&t[_]!==this.props[_])return!0;for(let _ in this.props)if(!(_ in t))return!0;return!1};function tn(t){return At(()=>a(t),[])}function nn(t){const n=Tt(t);n.current=t;Kt.__$f|=4;return At(()=>m(()=>n.current()),[])}function en(t){const n=Tt(t);n.current=t;$t(()=>w(()=>n.current()),[])}var _n=function(t,n,e,_){var i;n[0]=0;for(var o=1;o<n.length;o++){var r=n[o++],u=n[o]?(n[0]|=r?1:2,e[n[o++]]):n[++o];3===r?_[0]=u:4===r?_[1]=Object.assign(_[1]||{},u):5===r?(_[1]=_[1]||{})[n[++o]]=u:6===r?_[1][n[++o]]+=u+"":r?(i=t.apply(u,_n(t,u,e,["",null])),_.push(i),u[0]?n[0]|=2:(n[o-2]=0,n[o]=i)):_.push(u)}return _},on=new Map;function rn(t){var n=on.get(this);return n||(n=new Map,on.set(this,n)),(n=_n(this,n.get(t)||(n.set(t,n=function(t){for(var n,e,_=1,i="",o="",r=[0],u=function(t){1===_&&(t||(i=i.replace(/^\s*\n\s*|\s*\n\s*$/g,"")))?r.push(0,t,i):3===_&&(t||i)?(r.push(3,t,i),_=2):2===_&&"..."===i&&t?r.push(4,t,0):2===_&&i&&!t?r.push(5,0,!0,i):_>=5&&((i||!t&&5===_)&&(r.push(_,0,i,e),_=6),t&&(r.push(_,t,0,e),_=6)),i=""},f=0;f<t.length;f++){f&&(1===_&&u(),u(f));for(var l=0;l<t[f].length;l++)n=t[f][l],1===_?"<"===n?(u(),r=[r],_=3):i+=n:4===_?"--"===i&&">"===n?(_=1,i=""):i=n+i[0]:o?n===o?o="":i+=n:'"'===n||"'"===n?o=n:">"===n?(u(),_=1):_&&("="===n?(_=5,e=i,i=""):"/"===n&&(_<5||">"===t[f][l+1])?(u(),3===_&&(r=r[0]),_=r,(r=r[0]).push(2,0,_),_=0):" "===n||"\t"===n||"\n"===n||"\r"===n?(u(),_=2):i+=n),3===_&&"!--"===i&&(_=4,r=r[0])}return u(),r}(t)),n),arguments,[])).length>1?n:n[0]}var un=rn.bind(L);export{I as Component,j as Fragment,h as Signal,_ as batch,ht as cloneElement,m as computed,at as createContext,L as createElement,R as createRef,w as effect,L as h,un as html,ct as hydrate,U as isValidElement,C as options,st as render,a as signal,Y as toChildArray,u as untracked,Ft as useCallback,nn as useComputed,Mt as useContext,Wt as useDebugValue,$t as useEffect,Lt as useErrorBoundary,Ot as useId,Vt as useImperativeHandle,Dt as useLayoutEffect,At as useMemo,Nt as useReducer,Tt as useRef,tn as useSignal,en as useSignalEffect,Pt as useState};diff --git a/llama.cpp/server/public/json-schema-to-grammar.mjs b/llama.cpp/server/public/json-schema-to-grammar.mjsindex 3f1b255c..baa6761c 100644--- a/llama.cpp/server/public/json-schema-to-grammar.mjs+++ b/llama.cpp/server/public/json-schema-to-grammar.mjs@@ -1,25 +1,50 @@+// WARNING: This file was ported from json-schema-to-grammar.py, please fix bugs / add features there first.const SPACE_RULE = '" "?';const PRIMITIVE_RULES = {boolean: '("true" | "false") space',number: '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space',integer: '("-"? ([0-9] | [1-9] [0-9]*)) space',+ value: 'object | array | string | number | boolean',+ object: '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space',+ array: '"[" space ( value ("," space value)* )? "]" space',+ uuid: '"\\"" ' + [8, 4, 4, 4, 12].map(n => [...new Array(n)].map(_ => '[0-9a-fA-F]').join('')).join(' "-" ') + ' "\\"" space',string: ` "\\"" ([^"\\\\] |"\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]))* "\\"" space`,null: '"null" space',};+const OBJECT_RULE_NAMES = ['object', 'array', 'string', 'number', 'boolean', 'null', 'value'];++// TODO: support "uri", "email" string formats+const DATE_RULES = {+ 'date' : '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )',+ 'time' : '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )',+ 'date-time': 'date "T" time',+ 'date-string': '"\\"" date "\\"" space',+ 'time-string': '"\\"" time "\\"" space',+ 'date-time-string': '"\\"" date-time "\\"" space',+};++const RESERVED_NAMES = {'root': true, ...PRIMITIVE_RULES, ...DATE_RULES};const INVALID_RULE_CHARS_RE = /[^\dA-Za-z-]+/g;const GRAMMAR_LITERAL_ESCAPE_RE = /[\n\r"]/g;-const GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"'};+const GRAMMAR_RANGE_LITERAL_ESCAPE_RE = /[\n\r"\]\-\\]/g;+const GRAMMAR_LITERAL_ESCAPES = { '\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]' };++const NON_LITERAL_SET = new Set('|.()[]{}*+?');+const ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = new Set('[]()|{}*+?');export class SchemaConverter {- constructor(propOrder) {- this._propOrder = propOrder || {};- this._rules = new Map();- this._rules.set('space', SPACE_RULE);+ constructor(options) {+ this._propOrder = options.prop_order || {};+ this._allowFetch = options.allow_fetch || false;+ this._dotall = options.dotall || false;+ this._rules = {'space': SPACE_RULE};+ this._refs = {};+ this._refsBeingResolved = new Set();}_formatLiteral(literal) {@@ -30,83 +55,490 @@ export class SchemaConverter {return `"${escaped}"`;}+ _formatRangeChar(literal) {+ return JSON.stringify(literal).slice(1, -1).replace(+ GRAMMAR_RANGE_LITERAL_ESCAPE_RE,+ m => GRAMMAR_LITERAL_ESCAPES[m]+ );+ }+_addRule(name, rule) {let escName = name.replace(INVALID_RULE_CHARS_RE, '-');let key = escName;- if (this._rules.has(escName)) {- if (this._rules.get(escName) === rule) {+ if (escName in this._rules) {+ if (this._rules[escName] === rule) {return key;}let i = 0;- while (this._rules.has(`${escName}${i}`)) {+ while ((`${escName}${i}` in this._rules) && (this._rules[`${escName}${i}`] !== rule)) {i += 1;}key = `${escName}${i}`;}- this._rules.set(key, rule);+ this._rules[key] = rule;return key;}+ async resolveRefs(schema, url) {+ const visit = async (n) => {+ if (Array.isArray(n)) {+ return Promise.all(n.map(visit));+ } else if (typeof n === 'object' && n !== null) {+ let ref = n.$ref;+ let target;+ if (ref !== undefined && !this._refs[ref]) {+ if (ref.startsWith('https://')) {+ if (!this._allowFetch) {+ throw new Error('Fetching remote schemas is not allowed (use --allow-fetch for force)');+ }+ const fetch = (await import('node-fetch')).default;++ const fragSplit = ref.split('#');+ const baseUrl = fragSplit[0];++ target = this._refs[baseUrl];+ if (!target) {+ target = await this.resolveRefs(await fetch(ref).then(res => res.json()), baseUrl);+ this._refs[baseUrl] = target;+ }++ if (fragSplit.length === 1 || fragSplit[fragSplit.length - 1] === '') {+ return target;+ }+ } else if (ref.startsWith('#/')) {+ target = schema;+ ref = `${url}${ref}`;+ n.$ref = ref;+ } else {+ throw new Error(`Unsupported ref ${ref}`);+ }++ const selectors = ref.split('#')[1].split('/').slice(1);+ for (const sel of selectors) {+ if (!target || !(sel in target)) {+ throw new Error(`Error resolving ref ${ref}: ${sel} not in ${JSON.stringify(target)}`);+ }+ target = target[sel];+ }++ this._refs[ref] = target;+ } else {+ await Promise.all(Object.values(n).map(visit));+ }+ }++ return n;+ };++ return visit(schema);+ }++ _generateUnionRule(name, altSchemas) {+ return altSchemas+ .map((altSchema, i) => this.visit(altSchema, `${name ?? ''}${name ? '-' : 'alternative-'}${i}`))+ .join(' | ');+ }++ _visitPattern(pattern, name) {+ if (!pattern.startsWith('^') || !pattern.endsWith('$')) {+ throw new Error('Pattern must start with "^" and end with "$"');+ }+ pattern = pattern.slice(1, -1);+ const subRuleIds = {};++ let i = 0;+ const length = pattern.length;++ const getDot = () => {+ let rule;+ if (this._dotall) {+ rule = '[\\U00000000-\\U0010FFFF]';+ } else {+ // Accept any character... except \n and \r line break chars (\x0A and \xOD)+ rule = '[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]';+ }+ return this._addRule('dot', rule);+ };+++ const toRule = ([s, isLiteral]) => isLiteral ? "\"" + s + "\"" : s;++ const transform = () => {+ const start = i;+ // For each component of this sequence, store its string representation and whether it's a literal.+ // We only need a flat structure here to apply repetition operators to the last item, and+ // to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially+ // (GBNF's syntax is luckily very close to regular expressions!)+ const seq = [];++ const joinSeq = () => {+ const ret = [];+ for (const [isLiteral, g] of groupBy(seq, x => x[1])) {+ if (isLiteral) {+ ret.push([[...g].map(x => x[0]).join(''), true]);+ } else {+ ret.push(...g);+ }+ }+ if (ret.length === 1) {+ return ret[0];+ }+ return [ret.map(x => toRule(x)).join(' '), false];+ };++ while (i < length) {+ const c = pattern[i];+ if (c === '.') {+ seq.push([getDot(), false]);+ i += 1;+ } else if (c === '(') {+ i += 1;+ if (i < length) {+ if (pattern[i] === '?') {+ throw new Error(`Unsupported pattern syntax "${pattern[i]}" at index ${i} of /${pattern}/`);+ }+ }+ seq.push([`(${toRule(transform())})`, false]);+ } else if (c === ')') {+ i += 1;+ if (start <= 0 || pattern[start - 1] !== '(') {+ throw new Error(`Unbalanced parentheses; start = ${start}, i = ${i}, pattern = ${pattern}`);+ }+ return joinSeq();+ } else if (c === '[') {+ let squareBrackets = c;+ i += 1;+ while (i < length && pattern[i] !== ']') {+ if (pattern[i] === '\\') {+ squareBrackets += pattern.slice(i, i + 2);+ i += 2;+ } else {+ squareBrackets += pattern[i];+ i += 1;+ }+ }+ if (i >= length) {+ throw new Error(`Unbalanced square brackets; start = ${start}, i = ${i}, pattern = ${pattern}`);+ }+ squareBrackets += ']';+ i += 1;+ seq.push([squareBrackets, false]);+ } else if (c === '|') {+ seq.push(['|', false]);+ i += 1;+ } else if (c === '*' || c === '+' || c === '?') {+ seq[seq.length - 1] = [toRule(seq[seq.length - 1]) + c, false];+ i += 1;+ } else if (c === '{') {+ let curlyBrackets = c;+ i += 1;+ while (i < length && pattern[i] !== '}') {+ curlyBrackets += pattern[i];+ i += 1;+ }+ if (i >= length) {+ throw new Error(`Unbalanced curly brackets; start = ${start}, i = ${i}, pattern = ${pattern}`);+ }+ curlyBrackets += '}';+ i += 1;+ const nums = curlyBrackets.slice(1, -1).split(',').map(s => s.trim());+ let minTimes, maxTimes;+ if (nums.length === 1) {+ minTimes = parseInt(nums[0], 10);+ maxTimes = minTimes;+ } else {+ if (nums.length !== 2) {+ throw new Error(`Invalid quantifier ${curlyBrackets}`);+ }+ minTimes = nums[0] ? parseInt(nums[0], 10) : 0;+ maxTimes = nums[1] ? parseInt(nums[1], 10) : Infinity;+ }++ let [sub, subIsLiteral] = seq[seq.length - 1];++ if (minTimes === 0 && maxTimes === Infinity) {+ seq[seq.length - 1] = [`${sub}*`, false];+ } else if (minTimes === 0 && maxTimes === 1) {+ seq[seq.length - 1] = [`${sub}?`, false];+ } else if (minTimes === 1 && maxTimes === Infinity) {+ seq[seq.length - 1] = [`${sub}+`, false];+ } else {+ if (!subIsLiteral) {+ let id = subRuleIds[sub];+ if (id === undefined) {+ id = this._addRule(`${name}-${Object.keys(subRuleIds).length + 1}`, sub);+ subRuleIds[sub] = id;+ }+ sub = id;+ }++ const repeatedSub = Array.from({ length: minTimes }, () => subIsLiteral ? `"${sub.slice(1, -1).repeat(minTimes)}"` : sub);+ const optionalSub = maxTimes !== undefined ? Array.from({ length: maxTimes - minTimes }, () => `${sub}?`) : [`${sub}*`];+ seq[seq.length - 1] = [repeatedSub.concat(optionalSub).join(' '), false];+ }+ } else {+ let literal = '';+ while (i < length) {+ if (pattern[i] === '\\' && i < length - 1) {+ const next = pattern[i + 1];+ if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.has(next)) {+ i += 1;+ literal += pattern[i];+ i += 1;+ } else {+ literal += pattern.slice(i, i + 2);+ i += 2;+ }+ } else if (pattern[i] === '"') {+ literal += '\\"';+ i += 1;+ } else if (!NON_LITERAL_SET.has(pattern[i]) &&+ (i === length - 1 || literal === '' || pattern[i + 1] === '.' || !NON_LITERAL_SET.has(pattern[i+1]))) {+ literal += pattern[i];+ i += 1;+ } else {+ break;+ }+ }+ if (literal !== '') {+ seq.push([literal, true]);+ }+ }+ }++ return joinSeq();+ };++ return this._addRule(name, "\"\\\"\" " + toRule(transform()) + " \"\\\"\" space")+ }++ _resolveRef(ref) {+ let refName = ref.split('/').pop();+ if (!(refName in this._rules) && !this._refsBeingResolved.has(ref)) {+ this._refsBeingResolved.add(ref);+ const resolved = this._refs[ref];+ refName = this.visit(resolved, refName);+ this._refsBeingResolved.delete(ref);+ }+ return refName;+ }++ _generateConstantRule(value) {+ if (typeof value !== 'string') {+ throw new Error('Only string constants are supported, got ' + JSON.stringify(value));+ }+ return this._formatLiteral(value);+ }+visit(schema, name) {const schemaType = schema.type;- const ruleName = name || 'root';-- if (schema.oneOf || schema.anyOf) {- const rule = (schema.oneOf || schema.anyOf).map((altSchema, i) =>- this.visit(altSchema, `${name}${name ? "-" : ""}${i}`)- ).join(' | ');+ const schemaFormat = schema.format;+ const ruleName = name in RESERVED_NAMES ? name + '-' : name == '' ? 'root' : name;- return this._addRule(ruleName, rule);+ const ref = schema.$ref;+ if (ref !== undefined) {+ return this._addRule(ruleName, this._resolveRef(ref));+ } else if (schema.oneOf || schema.anyOf) {+ return this._addRule(ruleName, this._generateUnionRule(name, schema.oneOf || schema.anyOf));+ } else if (Array.isArray(schemaType)) {+ return this._addRule(ruleName, this._generateUnionRule(name, schemaType.map(t => ({ type: t }))));} else if ('const' in schema) {- return this._addRule(ruleName, this._formatLiteral(schema.const));+ if (typeof schema.const !== 'string') {+ throw new Error('Only string constants are supported, got ' + JSON.stringify(schema.const));+ }+ return this._addRule(ruleName, this._generateConstantRule(schema.const));} else if ('enum' in schema) {- const rule = schema.enum.map(v => this._formatLiteral(v)).join(' | ');+ const rule = schema.enum.map(v => this._generateConstantRule(v)).join(' | ');return this._addRule(ruleName, rule);- } else if (schemaType === 'object' && 'properties' in schema) {- // TODO: `required` keyword (from python implementation)- const propOrder = this._propOrder;- const propPairs = Object.entries(schema.properties).sort((a, b) => {- // sort by position in prop_order (if specified) then by key- const orderA = typeof propOrder[a[0]] === 'number' ? propOrder[a[0]] : Infinity;- const orderB = typeof propOrder[b[0]] === 'number' ? propOrder[b[0]] : Infinity;- return orderA - orderB || a[0].localeCompare(b[0]);- });-- let rule = '"{" space';- propPairs.forEach(([propName, propSchema], i) => {- const propRuleName = this.visit(propSchema, `${name}${name ? "-" : ""}${propName}`);- if (i > 0) {- rule += ' "," space';+ } else if ((schemaType === undefined || schemaType === 'object') &&+ ('properties' in schema ||+ ('additionalProperties' in schema && schema.additionalProperties !== true))) {+ const required = new Set(schema.required || []);+ const properties = Object.entries(schema.properties ?? {});+ return this._addRule(ruleName, this._buildObjectRule(properties, required, name, schema.additionalProperties));+ } else if ((schemaType === undefined || schemaType === 'object') && 'allOf' in schema) {+ const required = new Set();+ const properties = [];+ const addComponent = (compSchema, isRequired) => {+ const ref = compSchema.$ref;+ if (ref !== undefined) {+ compSchema = this._refs[ref];}- rule += ` ${this._formatLiteral(propName)} space ":" space ${propRuleName}`;- });- rule += ' "}" space';- return this._addRule(ruleName, rule);- } else if (schemaType === 'array' && 'items' in schema) {- // TODO `prefixItems` keyword (from python implementation)- const itemRuleName = this.visit(schema.items, `${name}${name ? "-" : ""}item`);- const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`;- return this._addRule(ruleName, rule);+ if ('properties' in compSchema) {+ for (const [propName, propSchema] of Object.entries(compSchema.properties)) {+ properties.push([propName, propSchema]);+ if (isRequired) {+ required.add(propName);+ }+ }+ }+ };++ for (const t of schema.allOf) {+ if ('anyOf' in t) {+ for (const tt of t.anyOf) {+ addComponent(tt, false);+ }+ } else {+ addComponent(t, true);+ }+ }++ return this._addRule(ruleName, this._buildObjectRule(properties, required, name, /* additionalProperties= */ false));+ } else if ((schemaType === undefined || schemaType === 'array') && ('items' in schema || 'prefixItems' in schema)) {+ const items = schema.items ?? schema.prefixItems;+ if (Array.isArray(items)) {+ return this._addRule(+ ruleName,+ '"[" space ' ++ items.map((item, i) => this.visit(item, `${name ?? ''}${name ? '-' : ''}tuple-${i}`)).join(' "," space ') ++ ' "]" space'+ );+ } else {+ const itemRuleName = this.visit(items, `${name ?? ''}${name ? '-' : ''}item`);+ const listItemOperator = `( "," space ${itemRuleName} )`;+ let successiveItems = '';+ let minItems = schema.minItems || 0;+ const maxItems = schema.maxItems;+ if (minItems > 0) {+ successiveItems = listItemOperator.repeat(minItems - 1);+ minItems--;+ }+ if (maxItems !== undefined && maxItems > minItems) {+ successiveItems += `${listItemOperator}?`.repeat(maxItems - minItems - 1);+ } else {+ successiveItems += `${listItemOperator}*`;+ }+ const rule = minItems === 0+ ? `"[" space ( ${itemRuleName} ${successiveItems} )? "]" space`+ : `"[" space ${itemRuleName} ${successiveItems} "]" space`;+ return this._addRule(ruleName, rule);+ }+ } else if ((schemaType === undefined || schemaType === 'string') && 'pattern' in schema) {+ return this._visitPattern(schema.pattern, ruleName);+ } else if ((schemaType === undefined || schemaType === 'string') && /^uuid[1-5]?$/.test(schema.format || '')) {+ return this._addRule(+ ruleName === 'root' ? 'root' : schemaFormat,+ PRIMITIVE_RULES['uuid'])+ } else if ((schemaType === undefined || schemaType === 'string') && schema.format in DATE_RULES) {+ for (const [t, r] of Object.entries(DATE_RULES)) {+ this._addRule(t, r);+ }+ return schemaFormat + '-string';+ } else if ((schemaType === 'object') || (Object.keys(schema).length === 0)) {+ for (const n of OBJECT_RULE_NAMES) {+ this._addRule(n, PRIMITIVE_RULES[n]);+ }+ return this._addRule(ruleName, 'object');} else {- if (!PRIMITIVE_RULES[schemaType]) {+ if (!(schemaType in PRIMITIVE_RULES)) {throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`);}- return this._addRule(- ruleName === 'root' ? 'root' : schemaType,- PRIMITIVE_RULES[schemaType]+ // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero+ return this._addRule(ruleName === 'root' ? 'root' : schemaType, PRIMITIVE_RULES[schemaType]);+ }+ }++ _buildObjectRule(properties, required, name, additionalProperties) {+ const propOrder = this._propOrder;+ // sort by position in prop_order (if specified) then by original order+ const sortedProps = properties.map(([k]) => k).sort((a, b) => {+ const orderA = propOrder[a] || Infinity;+ const orderB = propOrder[b] || Infinity;+ return orderA - orderB || properties.findIndex(([k]) => k === a) - properties.findIndex(([k]) => k === b);+ });++ const propKvRuleNames = {};+ for (const [propName, propSchema] of properties) {+ const propRuleName = this.visit(propSchema, `${name ?? ''}${name ? '-' : ''}${propName}`);+ propKvRuleNames[propName] = this._addRule(+ `${name ?? ''}${name ? '-' : ''}${propName}-kv`,+ `${this._formatLiteral(propName)} space ":" space ${propRuleName}`);}+ const requiredProps = sortedProps.filter(k => required.has(k));+ const optionalProps = sortedProps.filter(k => !required.has(k));++ if (typeof additionalProperties === 'object' || additionalProperties === true) {+ const subName = `${name ?? ''}${name ? '-' : ''}additional`;+ const valueRule = this.visit(additionalProperties === true ? {} : additionalProperties, `${subName}-value`);+ propKvRuleNames['*'] = this._addRule(+ `${subName}-kv`,+ `${this._addRule('string', PRIMITIVE_RULES['string'])} ":" space ${valueRule}`);+ optionalProps.push('*');+ }++ let rule = '"{" space ';+ rule += requiredProps.map(k => propKvRuleNames[k]).join(' "," space ');++ if (optionalProps.length > 0) {+ rule += ' (';+ if (requiredProps.length > 0) {+ rule += ' "," space ( ';+ }++ const getRecursiveRefs = (ks, firstIsOptional) => {+ const [k, ...rest] = ks;+ const kvRuleName = propKvRuleNames[k];+ let res;+ if (k === '*') {+ res = this._addRule(+ `${name ?? ''}${name ? '-' : ''}additional-kvs`,+ `${kvRuleName} ( "," space ` + kvRuleName + ` )*`+ )+ } else if (firstIsOptional) {+ res = `( "," space ${kvRuleName} )?`;+ } else {+ res = kvRuleName;+ }+ if (rest.length > 0) {+ res += ' ' + this._addRule(+ `${name ?? ''}${name ? '-' : ''}${k}-rest`,+ getRecursiveRefs(rest, true)+ );+ }+ return res;+ };++ rule += optionalProps.map((_, i) => getRecursiveRefs(optionalProps.slice(i), false)).join(' | ');+ if (requiredProps.length > 0) {+ rule += ' )';+ }+ rule += ' )?';+ }++ rule += ' "}" space';++ return rule;}formatGrammar() {let grammar = '';- this._rules.forEach((rule, name) => {+ for (const [name, rule] of Object.entries(this._rules).sort(([a], [b]) => a.localeCompare(b))) {grammar += `${name} ::= ${rule}\n`;- });+ }return grammar;}}++// Helper function to group elements by a key function+function* groupBy(iterable, keyFn) {+ let lastKey = null;+ let group = [];+ for (const element of iterable) {+ const key = keyFn(element);+ if (lastKey !== null && key !== lastKey) {+ yield [lastKey, group];+ group = [];+ }+ group.push(element);+ lastKey = key;+ }+ if (group.length > 0) {+ yield [lastKey, group];+ }+}diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex e87e9880..914482ab 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -1,152 +1,141 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "utils.h"+#include "macsandbox.h"+#include "llama.cpp/common.h"#include "llama.cpp/llama.h"#include "llama.cpp/grammar-parser.h"-#include "llama.cpp/llava/llava.h"-#include "llama.cpp/stb_image.h"-#include "utils.h"-#include "oai.h"-#include "macsandbox.h"+#include "llama.cpp/json-schema-to-grammar.h"// increase max payload length to allow use of larger context size#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576#include "httplib.h"-#include "json.h"+#include "llama.cpp/json.h"-#include <cstddef>-#include <thread>+#include <atomic>#include <chrono>#include <condition_variable>-#include <atomic>+#include <cstddef>+#include <set>+#include <mutex>+#include <thread>#include <signal.h>+#include <memory>#include <libc/calls/pledge.h>#include <tool/args/args.h>#include <libc/dce.h>using json = nlohmann::json;-struct server_params-{- std::string hostname = "127.0.0.1";- std::vector<std::string> api_keys;- std::string public_path = "/zip/llama.cpp/server/public";- std::string chat_template = "";- int32_t port = 8080;- int32_t read_timeout = 600;- int32_t write_timeout = 600;- bool nobrowser = false;- bool slots_endpoint = true;- bool metrics_endpoint = false;-};-bool server_verbose = false;bool server_log_json = true;-static size_t common_part(const std::vector<llama_token> &a, const std::vector<llama_token> &b)-{- size_t i;- for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++)- {- }- return i;-}+enum stop_type {+ STOP_TYPE_FULL,+ STOP_TYPE_PARTIAL,+};-enum stop_type-{- STOP_FULL,- STOP_PARTIAL,+enum slot_state {+ SLOT_STATE_IDLE,+ SLOT_STATE_PROCESSING,};-static bool ends_with(const std::string &str, const std::string &suffix)-{- return str.size() >= suffix.size() &&- 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);-}+enum slot_command {+ SLOT_COMMAND_NONE,+ SLOT_COMMAND_LOAD_PROMPT,+ SLOT_COMMAND_RELEASE,+};-static size_t find_partial_stop_string(const std::string &stop,- const std::string &text)-{- if (!text.empty() && !stop.empty())- {- const char text_last_char = text.back();- for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--)- {- if (stop[char_index] == text_last_char)- {- const std::string current_partial = stop.substr(0, char_index + 1);- if (ends_with(text, current_partial))- {- return text.size() - char_index - 1;- }- }- }- }- return std::string::npos;-}+enum server_state {+ SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet+ SERVER_STATE_READY, // Server is ready and model is loaded+ SERVER_STATE_ERROR // An error occurred, load_model failed+};-// TODO: reuse llama_detokenize-template <class Iter>-static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end)-{- std::string ret;- for (; begin != end; ++begin)- {- ret += llama_token_to_piece(ctx, *begin);- }- return ret;-}+enum server_task_type {+ SERVER_TASK_TYPE_COMPLETION,+ SERVER_TASK_TYPE_CANCEL,+ SERVER_TASK_TYPE_NEXT_RESPONSE,+ SERVER_TASK_TYPE_METRICS+};-// format incomplete utf-8 multibyte character for output-static std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token)-{- std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token);- // if the size is 1 and first bit is 1, meaning it's a partial character- // (size > 1 meaning it's already a known token)- if (out.size() == 1 && (out[0] & 0x80) == 0x80)- {- std::stringstream ss;- ss << std::hex << (out[0] & 0xff);- std::string res(ss.str());- out = "byte: \\x" + res;- }- return out;-}+struct server_task {+ int id = -1; // to be filled by server_queue+ int id_multi = -1;+ int id_target = -1;-// convert a vector of completion_token_output to json-static json probs_vector_to_json(const llama_context *ctx, const std::vector<completion_token_output> &probs)-{- json out = json::array();- for (const auto &prob : probs)- {- json probs_for_token = json::array();- for (const auto &p : prob.probs)- {- std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);- probs_for_token.push_back(json- {- {"tok_str", tok_str},- {"prob", p.prob},- });- }- std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);- out.push_back(json{- {"content", tok_str},- {"probs", probs_for_token},- });- }- return out;-}+ server_task_type type;+ json data;++ bool infill = false;+ bool embedding = false;+};++struct server_task_result {+ int id = -1;+ int id_multi = -1;++ json data;++ bool stop;+ bool error;+};++struct server_task_multi {+ int id = -1;++ std::set<int> subtasks_remaining;+ std::vector<server_task_result> results;+};++struct slot_params {+ bool stream = true;+ bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt++ uint32_t seed = -1; // RNG seed+ int32_t n_keep = 0; // number of tokens to keep from initial prompt+ int32_t n_predict = -1; // new tokens to predict++ std::vector<std::string> antiprompt;++ json input_prefix;+ json input_suffix;+};++struct server_params {+ int32_t port = 8080;+ int32_t read_timeout = 600;+ int32_t write_timeout = 600;+ int32_t n_threads_http = -1;-struct llama_client_slot-{+ std::string hostname = "127.0.0.1";+ std::string public_path = "/zip/llama.cpp/server/public";+ std::string chat_template = "";+ std::string system_prompt = "";++ std::vector<std::string> api_keys;++#ifdef CPPHTTPLIB_OPENSSL_SUPPORT+ std::string ssl_key_file = "";+ std::string ssl_cert_file = "";+#endif++ bool slots_endpoint = true;+ bool metrics_endpoint = false;++ bool nobrowser = false;+};++struct server_slot {int id;- int task_id = -1;+ int id_task = -1;+ int id_multi = -1;struct slot_params params;- slot_state state = IDLE;- slot_command command = NONE;+ slot_state state = SLOT_STATE_IDLE;+ slot_command command = SLOT_COMMAND_NONE;// used to determine the slot that has been used the longestint64_t t_last_used = -1;@@ -157,33 +146,38 @@ struct llama_client_slotint32_t n_decoded = 0;int32_t n_remaining = -1;int32_t i_batch = -1;- int32_t n_predict = -1;+ int32_t n_predict = -1; // TODO: disambiguate from params.n_predict- int32_t num_prompt_tokens = 0;- int32_t num_prompt_tokens_processed = 0;+ int32_t n_prompt_tokens = 0;+ int32_t n_prompt_tokens_processed = 0;json prompt;++ // when a task is submitted, we first tokenize the prompt and store it here+ std::vector<llama_token> prompt_tokens;+std::string generated_text;- llama_token sampled;std::vector<llama_token> cache_tokens;std::vector<completion_token_output> generated_token_probs;- bool infill = false;- bool embedding = false;+ bool infill = false;+ bool embedding = false;bool has_next_token = true;- bool truncated = false;- bool stopped_eos = false;- bool stopped_word = false;- bool stopped_limit = false;+ bool truncated = false;+ bool stopped_eos = false;+ bool stopped_word = false;+ bool stopped_limit = false;bool oaicompat = false;- std::string oaicompat_model;+ std::string oaicompat_model;std::string stopping_word;// sampling+ llama_token sampled;struct llama_sampling_params sparams;- llama_sampling_context *ctx_sampling = nullptr;+ llama_sampling_context * ctx_sampling = nullptr;+ json json_schema;int32_t ga_i = 0; // group-attention stateint32_t ga_n = 1; // group-attention factor@@ -191,65 +185,44 @@ struct llama_client_slotint32_t n_past_se = 0; // self-extend- // multimodal- std::vector<slot_image> images;-// stats- size_t sent_count = 0;- size_t sent_token_probs_index = 0;+ size_t n_sent_text = 0; // number of sent text character+ size_t n_sent_token_probs = 0;int64_t t_start_process_prompt;- int64_t t_start_genereration;+ int64_t t_start_generation;double t_prompt_processing; // msdouble t_token_generation; // ms- // multitasks- int multitask_id = -1;-void reset() {- num_prompt_tokens = 0;- generated_text = "";- truncated = false;- stopped_eos = false;- stopped_word = false;- stopped_limit = false;- stopping_word = "";- n_past = 0;- sent_count = 0;- sent_token_probs_index = 0;- infill = false;- ga_i = 0;- n_past_se = 0;+ n_prompt_tokens = 0;+ generated_text = "";+ truncated = false;+ stopped_eos = false;+ stopped_word = false;+ stopped_limit = false;+ stopping_word = "";+ n_past = 0;+ n_sent_text = 0;+ n_sent_token_probs = 0;+ infill = false;+ ga_i = 0;+ n_past_se = 0;generated_token_probs.clear();-- for (slot_image & img : images)- {- free(img.image_embedding);- if (img.img_data) {- clip_image_u8_free(img.img_data);- }- img.prefix_prompt = "";- }-- images.clear();}bool has_budget(gpt_params &global_params) {- if (params.n_predict == -1 && global_params.n_predict == -1)- {+ if (params.n_predict == -1 && global_params.n_predict == -1) {return true; // limitless}n_remaining = -1;- if (params.n_predict != -1)- {+ if (params.n_predict != -1) {n_remaining = params.n_predict - n_decoded;- }- else if (global_params.n_predict != -1)- {+ } else if (global_params.n_predict != -1) {n_remaining = global_params.n_predict - n_decoded;}@@ -257,37 +230,33 @@ struct llama_client_slot}bool available() const {- return state == IDLE && command == NONE;+ return state == SLOT_STATE_IDLE && command == SLOT_COMMAND_NONE;}bool is_processing() const {- return (state == IDLE && command == LOAD_PROMPT) || state == PROCESSING;+ return (state == SLOT_STATE_IDLE && command == SLOT_COMMAND_LOAD_PROMPT) || state == SLOT_STATE_PROCESSING;}- void add_token_string(const completion_token_output &token) {- if (command == RELEASE)- {+ void add_token_string(const completion_token_output & token) {+ if (command == SLOT_COMMAND_RELEASE) {return;}- cache_tokens.push_back(token.tok);generated_token_probs.push_back(token);}void release() {- if (state == PROCESSING)- {- t_token_generation = (ggml_time_us() - t_start_genereration) / 1e3;- command = RELEASE;+ if (state == SLOT_STATE_PROCESSING) {+ t_token_generation = (ggml_time_us() - t_start_generation) / 1e3;+ command = SLOT_COMMAND_RELEASE;}}- json get_formated_timings() {- return json- {- {"prompt_n", num_prompt_tokens_processed},+ json get_formated_timings() const {+ return json {+ {"prompt_n", n_prompt_tokens_processed},{"prompt_ms", t_prompt_processing},- {"prompt_per_token_ms", t_prompt_processing / num_prompt_tokens_processed},- {"prompt_per_second", 1e3 / t_prompt_processing * num_prompt_tokens_processed},+ {"prompt_per_token_ms", t_prompt_processing / n_prompt_tokens_processed},+ {"prompt_per_second", 1e3 / t_prompt_processing * n_prompt_tokens_processed},{"predicted_n", n_decoded},{"predicted_ms", t_token_generation},@@ -296,40 +265,74 @@ struct llama_client_slot};}+ size_t find_stopping_strings(const std::string & text, const size_t last_token_size, const stop_type type) {+ size_t stop_pos = std::string::npos;++ for (const std::string & word : params.antiprompt) {+ size_t pos;++ if (type == STOP_TYPE_FULL) {+ const size_t tmp = word.size() + last_token_size;+ const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;++ pos = text.find(word, from_pos);+ } else {+ pos = find_partial_stop_string(word, text);+ }++ if (pos != std::string::npos && (stop_pos == std::string::npos || pos < stop_pos)) {+ if (type == STOP_TYPE_FULL) {+ stopped_word = true;+ stopping_word = word;+ has_next_token = false;+ }+ stop_pos = pos;+ }+ }++ return stop_pos;+ }+void print_timings() const {- char buffer[512];- double t_token = t_prompt_processing / num_prompt_tokens_processed;- double n_tokens_second = 1e3 / t_prompt_processing * num_prompt_tokens_processed;- sprintf(buffer, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)",- t_prompt_processing, num_prompt_tokens_processed,+ char buffer[512];++ double t_token = t_prompt_processing / n_prompt_tokens_processed;+ double n_tokens_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;++ snprintf(buffer, 512, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)",+ t_prompt_processing, n_prompt_tokens_processed,t_token, n_tokens_second);+LOG_INFO(buffer, {- {"slot_id", id},- {"task_id", task_id},- {"t_prompt_processing", t_prompt_processing},- {"num_prompt_tokens_processed", num_prompt_tokens_processed},- {"t_token", t_token},- {"n_tokens_second", n_tokens_second},+ {"id_slot", id},+ {"id_task", id_task},+ {"t_prompt_processing", t_prompt_processing},+ {"n_prompt_tokens_processed", n_prompt_tokens_processed},+ {"t_token", t_token},+ {"n_tokens_second", n_tokens_second},});t_token = t_token_generation / n_decoded;n_tokens_second = 1e3 / t_token_generation * n_decoded;- sprintf(buffer, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)",++ snprintf(buffer, 512, "generation eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)",t_token_generation, n_decoded,t_token, n_tokens_second);+LOG_INFO(buffer, {- {"slot_id", id},- {"task_id", task_id},+ {"id_slot", id},+ {"id_task", id_task},{"t_token_generation", t_token_generation},{"n_decoded", n_decoded},{"t_token", t_token},{"n_tokens_second", n_tokens_second},});- sprintf(buffer, " total time = %10.2f ms", t_prompt_processing + t_token_generation);+ snprintf(buffer, 512, " total time = %10.2f ms", t_prompt_processing + t_token_generation);+LOG_INFO(buffer, {- {"slot_id", id},- {"task_id", task_id},+ {"id_slot", id},+ {"id_task", id_task},{"t_prompt_processing", t_prompt_processing},{"t_token_generation", t_token_generation},{"t_total", t_prompt_processing + t_token_generation},@@ -337,29 +340,36 @@ struct llama_client_slot}};-struct llama_metrics {+struct server_metrics {+ int64_t t_start = 0;+uint64_t n_prompt_tokens_processed_total = 0;+ uint64_t t_prompt_processing_total = 0;uint64_t n_tokens_predicted_total = 0;+ uint64_t t_tokens_generation_total = 0;uint64_t n_prompt_tokens_processed = 0;uint64_t t_prompt_processing = 0;- uint64_t n_tokens_predicted = 0;- uint64_t t_tokens_generation = 0;-+ uint64_t n_tokens_predicted = 0;+ uint64_t t_tokens_generation = 0;- void on_prompt_eval(const llama_client_slot &slot) {- n_prompt_tokens_processed_total += slot.num_prompt_tokens_processed;-- n_prompt_tokens_processed += slot.num_prompt_tokens_processed;- t_prompt_processing += slot.t_prompt_processing;+ void init() {+ t_start = ggml_time_us();}- void on_prediction(const llama_client_slot &slot) {- n_tokens_predicted_total += slot.n_decoded;+ void on_prompt_eval(const server_slot & slot) {+ n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed;+ n_prompt_tokens_processed += slot.n_prompt_tokens_processed;+ t_prompt_processing += slot.t_prompt_processing;+ t_prompt_processing_total += slot.t_prompt_processing;+ }- n_tokens_predicted += slot.n_decoded;- t_tokens_generation += slot.t_token_generation;+ void on_prediction(const server_slot & slot) {+ n_tokens_predicted_total += slot.n_decoded;+ n_tokens_predicted += slot.n_decoded;+ t_tokens_generation += slot.t_token_generation;+ t_tokens_generation_total += slot.t_token_generation;}void reset_bucket() {@@ -370,23 +380,261 @@ struct llama_metrics {}};-struct llama_server_context-{- llama_model *model = nullptr;- llama_context *ctx = nullptr;+struct server_queue {+ int id = 0;+ bool running;++ // queues+ std::vector<server_task> queue_tasks;+ std::vector<server_task> queue_tasks_deferred;++ std::vector<server_task_multi> queue_multitasks;++ std::mutex mutex_tasks;+ std::condition_variable condition_tasks;++ // callback functions+ std::function<void(server_task &)> callback_new_task;+ std::function<void(server_task_multi &)> callback_finish_multitask;+ std::function<void(void)> callback_update_slots;++ // Add a new task to the end of the queue+ int post(server_task task) {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (task.id == -1) {+ task.id = id++;+ LOG_VERBOSE("new task id", {{"new_id", task.id}});+ }+ queue_tasks.push_back(std::move(task));+ condition_tasks.notify_one();+ return task.id;+ }++ // Add a new task, but defer until one slot is available+ void defer(server_task task) {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ queue_tasks_deferred.push_back(std::move(task));+ }++ // Get the next id for creating anew task+ int get_new_id() {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ int new_id = id++;+ LOG_VERBOSE("new task id", {{"new_id", new_id}});+ return new_id;+ }++ // Register function to process a new task+ void on_new_task(std::function<void(server_task &)> callback) {+ callback_new_task = std::move(callback);+ }++ // Register function to process a multitask when it is finished+ void on_finish_multitask(std::function<void(server_task_multi&)> callback) {+ callback_finish_multitask = std::move(callback);+ }++ // Register the function to be called when all slots data is ready to be processed+ void on_update_slots(std::function<void(void)> callback) {+ callback_update_slots = std::move(callback);+ }++ // Call when the state of one slot is changed+ void notify_slot_changed() {+ // move deferred tasks back to main loop+ std::unique_lock<std::mutex> lock(mutex_tasks);+ for (auto & task : queue_tasks_deferred) {+ queue_tasks.push_back(std::move(task));+ }+ queue_tasks_deferred.clear();+ }++ // end the start_loop routine+ void terminate() {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ running = false;+ condition_tasks.notify_all();+ }++ /**+ * Main loop consists of these steps:+ * - Wait until a new task arrives+ * - Process the task (i.e. maybe copy data into slot)+ * - Check if multitask is finished+ * - Update all slots+ */+ void start_loop() {+ running = true;++ while (true) {+ LOG_VERBOSE("new task may arrive", {});++ while (true) {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (queue_tasks.empty()) {+ lock.unlock();+ break;+ }+ server_task task = queue_tasks.front();+ queue_tasks.erase(queue_tasks.begin());+ lock.unlock();+ LOG_VERBOSE("callback_new_task", {{"id_task", task.id}});+ callback_new_task(task);+ }++ LOG_VERBOSE("update_multitasks", {});++ // check if we have any finished multitasks+ auto queue_iterator = queue_multitasks.begin();+ while (queue_iterator != queue_multitasks.end()) {+ if (queue_iterator->subtasks_remaining.empty()) {+ // all subtasks done == multitask is done+ server_task_multi current_multitask = *queue_iterator;+ callback_finish_multitask(current_multitask);+ // remove this multitask+ queue_iterator = queue_multitasks.erase(queue_iterator);+ } else {+ ++queue_iterator;+ }+ }++ // all tasks in the current loop is processed, slots data is now ready+ LOG_VERBOSE("callback_update_slots", {});++ callback_update_slots();++ LOG_VERBOSE("wait for new task", {});+ {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (queue_tasks.empty()) {+ if (!running) {+ LOG_VERBOSE("ending start_loop", {});+ return;+ }+ condition_tasks.wait(lock, [&]{+ return (!queue_tasks.empty() || !running);+ });+ }+ }+ }+ }++ //+ // functions to manage multitasks+ //++ // add a multitask by specifying the id of all subtask (subtask is a server_task)+ void add_multitask(int id_multi, std::vector<int> & sub_ids) {+ std::lock_guard<std::mutex> lock(mutex_tasks);+ server_task_multi multi;+ multi.id = id_multi;+ std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));+ queue_multitasks.push_back(multi);+ }++ // updatethe remaining subtasks, while appending results to multitask+ void update_multitask(int id_multi, int id_sub, server_task_result & result) {+ std::lock_guard<std::mutex> lock(mutex_tasks);+ for (auto & multitask : queue_multitasks) {+ if (multitask.id == id_multi) {+ multitask.subtasks_remaining.erase(id_sub);+ multitask.results.push_back(result);+ }+ }+ }+};++struct server_response {+ typedef std::function<void(int, int, server_task_result &)> callback_multitask_t;+ callback_multitask_t callback_update_multitask;++ // for keeping track of all tasks waiting for the result+ std::set<int> waiting_task_ids;++ // the main result queue+ std::vector<server_task_result> queue_results;++ std::mutex mutex_results;+ std::condition_variable condition_results;++ // add the id_task to the list of tasks waiting for response+ void add_waiting_task_id(int id_task) {+ LOG_VERBOSE("waiting for task id", {{"id_task", id_task}});++ std::unique_lock<std::mutex> lock(mutex_results);+ waiting_task_ids.insert(id_task);+ }++ // when the request is finished, we can remove task associated with it+ void remove_waiting_task_id(int id_task) {+ LOG_VERBOSE("remove waiting for task id", {{"id_task", id_task}});++ std::unique_lock<std::mutex> lock(mutex_results);+ waiting_task_ids.erase(id_task);+ }++ // This function blocks the thread until there is a response for this id_task+ server_task_result recv(int id_task) {+ while (true) {+ std::unique_lock<std::mutex> lock(mutex_results);+ condition_results.wait(lock, [&]{+ return !queue_results.empty();+ });++ for (int i = 0; i < (int) queue_results.size(); i++) {+ if (queue_results[i].id == id_task) {+ assert(queue_results[i].id_multi == -1);+ server_task_result res = queue_results[i];+ queue_results.erase(queue_results.begin() + i);+ return res;+ }+ }+ }++ // should never reach here+ }++ // Register the function to update multitask+ void on_multitask_update(callback_multitask_t callback) {+ callback_update_multitask = std::move(callback);+ }++ // Send a new result to a waiting id_task+ void send(server_task_result result) {+ LOG_VERBOSE("send new result", {{"id_task", result.id}});++ std::unique_lock<std::mutex> lock(mutex_results);+ for (const auto & id_task : waiting_task_ids) {+ // LOG_TEE("waiting task id %i \n", id_task);+ // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result+ if (result.id_multi == id_task) {+ LOG_VERBOSE("callback_update_multitask", {{"id_task", id_task}});+ callback_update_multitask(id_task, result.id, result);+ continue;+ }++ if (result.id == id_task) {+ LOG_VERBOSE("queue_results.push_back", {{"id_task", id_task}});+ queue_results.push_back(result);+ condition_results.notify_all();+ return;+ }+ }+ }+};- clip_ctx *clp_ctx = nullptr;+struct server_context {+ llama_model * model = nullptr;+ llama_context * ctx = nullptr;gpt_params params;llama_batch batch;- bool multimodal = false;- bool clean_kv_cache = true;- bool all_slots_are_idle = false;- bool add_bos_token = true;+ bool clean_kv_cache = true;+ bool add_bos_token = true;- int32_t n_ctx; // total context for all clients / slots+ int32_t n_ctx; // total context for all clients / slots// system promptbool system_need_update = false;@@ -398,63 +646,39 @@ struct llama_server_contextstd::string name_assistant;// slots / clients- std::vector<llama_client_slot> slots;+ std::vector<server_slot> slots;json default_generation_settings_for_props;- llama_server_queue queue_tasks;- llama_server_response queue_results;+ server_queue queue_tasks;+ server_response queue_results;- llama_metrics metrics;+ server_metrics metrics;- ~llama_server_context()- {- if (ctx)- {+ ~server_context() {+ if (ctx) {llama_free(ctx);ctx = nullptr;}- if (model)- {++ if (model) {llama_free_model(model);model = nullptr;}}- bool load_model(const gpt_params ¶ms_)- {+ bool load_model(const gpt_params & params_) {params = params_;- if (!params.mmproj.empty()) {- multimodal = true;- LOG_INFO("Multi Modal Mode Enabled", {});- clp_ctx = clip_model_load(params.mmproj.c_str(), /*verbosity=*/ 1);- if(clp_ctx == nullptr) {- LOG_ERROR("unable to load clip model", {{"model", params.mmproj}});- return false;- }- if (params.n_ctx < 2048) { // request larger context for the image embedding- params.n_ctx = 2048;- }- }+ // dedicate one sequence to the system prompt+ params.n_parallel += 1;std::tie(model, ctx) = llama_init_from_gpt_params(params);- if (model == nullptr)- {+ params.n_parallel -= 1; // but be sneaky about it+ if (model == nullptr) {LOG_ERROR("unable to load model", {{"model", params.model}});return false;}- if (multimodal) {- const int n_embd_clip = clip_n_mmproj_embd(clp_ctx);- const int n_embd_llm = llama_n_embd(model);- if (n_embd_clip != n_embd_llm) {- LOG_TEE("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_embd_clip, n_embd_llm);- llama_free(ctx);- llama_free_model(model);- return false;- }- }-n_ctx = llama_n_ctx(ctx);add_bos_token = llama_should_add_bos_token(model);@@ -462,33 +686,28 @@ struct llama_server_contextreturn true;}- void validate_model_chat_template(server_params & sparams) {+ bool validate_model_chat_template() const {llama_chat_message chat[] = {{"user", "test"}};- std::vector<char> buf(1);- int res = llama_chat_apply_template(model, nullptr, chat, 1, true, buf.data(), buf.size());- if (res < 0) {- LOG_ERROR("The chat template comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {});- sparams.chat_template = "<|im_start|>"; // llama_chat_apply_template only checks if <|im_start|> exist in the template- }- }- void initialize() {- // create slots- all_slots_are_idle = true;+ const int res = llama_chat_apply_template(model, nullptr, chat, 1, true, nullptr, 0);++ return res > 0;+ }+ void init() {const int32_t n_ctx_slot = n_ctx / params.n_parallel;LOG_INFO("initializing slots", {{"n_slots", params.n_parallel}});- for (int i = 0; i < params.n_parallel; i++)- {- llama_client_slot slot;++ for (int i = 0; i < params.n_parallel; i++) {+ server_slot slot;slot.id = i;slot.n_ctx = n_ctx_slot;slot.n_predict = params.n_predict;LOG_INFO("new slot", {- {"slot_id", slot.id},+ {"id_slot", slot.id},{"n_ctx_slot", slot.n_ctx}});@@ -496,15 +715,15 @@ struct llama_server_contextconst int ga_w = params.grp_attn_w;if (ga_n != 1) {- GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT- GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT+ GGML_ASSERT(ga_n > 0 && "ga_n must be positive"); // NOLINT+ GGML_ASSERT(ga_w % ga_n == 0 && "ga_w must be a multiple of ga_n"); // NOLINT//GGML_ASSERT(n_ctx_train % ga_w == 0 && "n_ctx_train must be a multiple of ga_w"); // NOLINT//GGML_ASSERT(n_ctx >= n_ctx_train * ga_n && "n_ctx must be at least n_ctx_train * ga_n"); // NOLINTLOG_INFO("slot self-extend", {- {"slot_id", slot.id},- {"ga_n", ga_n},- {"ga_w", ga_w}+ {"id_slot", slot.id},+ {"ga_n", ga_n},+ {"ga_w", ga_w}});}@@ -520,11 +739,18 @@ struct llama_server_contextdefault_generation_settings_for_props = get_formated_generation(slots.front());default_generation_settings_for_props["seed"] = -1;- batch = llama_batch_init(n_ctx, 0, params.n_parallel);+ // the update_slots() logic will always submit a maximum of n_batch tokens+ // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used)+ {+ const int32_t n_batch = llama_n_batch(ctx);++ batch = llama_batch_init(n_batch, 0, params.n_parallel);+ }++ metrics.init();}- std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const- {+ std::vector<llama_token> tokenize(const json & json_prompt, bool add_bos) const {// TODO: currently, we tokenize using special tokens by default// this is not always correct (see https://github.com/ggerganov/llama.cpp/pull/4160#issuecomment-1824826216)// but it's better compared to completely ignoring ChatML and other chat templates@@ -534,38 +760,30 @@ struct llama_server_context// or the first element of the json_prompt array is a string.std::vector<llama_token> prompt_tokens;- if (json_prompt.is_array())- {+ if (json_prompt.is_array()) {bool first = true;- for (const auto& p : json_prompt)- {- if (p.is_string())- {+ for (const auto & p : json_prompt) {+ if (p.is_string()) {auto s = p.template get<std::string>();+std::vector<llama_token> p;- if (first)- {+ if (first) {p = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);first = false;- }- else- {+ } else {p = ::llama_tokenize(ctx, s, false, TMP_FORCE_SPECIAL);}+prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());- }- else- {- if (first)- {+ } else {+ if (first) {first = false;}+prompt_tokens.push_back(p.template get<llama_token>());}}- }- else- {+ } else {auto s = json_prompt.template get<std::string>();prompt_tokens = ::llama_tokenize(ctx, s, add_bos, TMP_FORCE_SPECIAL);}@@ -573,19 +791,18 @@ struct llama_server_contextreturn prompt_tokens;}- llama_client_slot* get_slot(int id) {+ server_slot * get_slot(int id) {int64_t t_last = ggml_time_us();- llama_client_slot *last_used = nullptr;- for (llama_client_slot & slot : slots)- {- if (slot.id == id && slot.available())- {+ server_slot * last_used = nullptr;++ for (server_slot & slot : slots) {+ if (slot.id == id && slot.available()) {return &slot;}- if (slot.available() && slot.t_last_used < t_last)- {+ // among all available slots, find the one that has been least recently used+ if (slot.available() && slot.t_last_used < t_last) {last_used = &slot;t_last = slot.t_last_used;}@@ -594,295 +811,235 @@ struct llama_server_contextreturn last_used;}- bool launch_slot_with_data(llama_client_slot* &slot, json data) {+ bool launch_slot_with_task(server_slot & slot, const server_task & task) {slot_params default_params;llama_sampling_params default_sparams;+ auto & data = task.data;if (data.count("__oaicompat") != 0) {- slot->oaicompat = true;- slot->oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));+ slot.oaicompat = true;+ slot.oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));+ } else {+ slot.oaicompat = false;+ slot.oaicompat_model = "";+ }++ slot.params.stream = json_value(data, "stream", false);+ slot.params.cache_prompt = json_value(data, "cache_prompt", false);+ slot.params.n_predict = json_value(data, "n_predict", default_params.n_predict);+ slot.sparams.top_k = json_value(data, "top_k", default_sparams.top_k);+ slot.sparams.top_p = json_value(data, "top_p", default_sparams.top_p);+ slot.sparams.min_p = json_value(data, "min_p", default_sparams.min_p);+ slot.sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);+ slot.sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);+ slot.sparams.temp = json_value(data, "temperature", default_sparams.temp);+ slot.sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);+ slot.sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);+ slot.sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);+ slot.sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);+ slot.sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);+ slot.sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);+ slot.sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);+ slot.sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);+ slot.sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);+ slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);+ slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep);+ slot.params.seed = json_value(data, "seed", default_params.seed);+ if (data.contains("json_schema") && !data.contains("grammar")) {+ try {+ auto schema = json_value(data, "json_schema", json::object());+ slot.sparams.grammar = json_schema_to_grammar(schema);+ } catch (const std::exception & e) {+ send_error(task, std::string("\"json_schema\": ") + e.what(), ERROR_TYPE_INVALID_REQUEST);+ return false;+ }} else {- slot->oaicompat = false;- slot->oaicompat_model = "";- }-- slot->params.stream = json_value(data, "stream", false);- slot->params.cache_prompt = json_value(data, "cache_prompt", false);- slot->params.n_predict = json_value(data, "n_predict", default_params.n_predict);- slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);- slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);- slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);- slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);- slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);- slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);- slot->sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range);- slot->sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent);- slot->sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n);- slot->sparams.penalty_repeat = json_value(data, "repeat_penalty", default_sparams.penalty_repeat);- slot->sparams.penalty_freq = json_value(data, "frequency_penalty", default_sparams.penalty_freq);- slot->sparams.penalty_present = json_value(data, "presence_penalty", default_sparams.penalty_present);- slot->sparams.mirostat = json_value(data, "mirostat", default_sparams.mirostat);- slot->sparams.mirostat_tau = json_value(data, "mirostat_tau", default_sparams.mirostat_tau);- slot->sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);- slot->sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);- slot->params.n_keep = json_value(data, "n_keep", slot->params.n_keep);- slot->params.seed = json_value(data, "seed", default_params.seed);- slot->sparams.grammar = json_value(data, "grammar", default_sparams.grammar);- slot->sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);- slot->sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);-- if (slot->n_predict > 0 && slot->params.n_predict > slot->n_predict) {+ slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar);+ }+ slot.sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs);+ slot.sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep);++ if (slot.params.cache_prompt && slot.ga_n != 1) {+ LOG_WARNING("cache_prompt is not supported with group-attention", {});+ slot.params.cache_prompt = false;+ }++ if (slot.n_predict > 0 && slot.params.n_predict > slot.n_predict) {// Might be better to reject the request with a 400 ?LOG_WARNING("Max tokens to predict exceeds server configuration", {- {"params.n_predict", slot->params.n_predict},- {"slot.n_predict", slot->n_predict},+ {"params.n_predict", slot.params.n_predict},+ {"slot.n_predict", slot.n_predict},});- slot->params.n_predict = slot->n_predict;+ slot.params.n_predict = slot.n_predict;}// infill- if (data.count("input_prefix") != 0)+ slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix);+ slot.params.input_suffix = json_value(data, "input_suffix", default_params.input_suffix);++ // get prompt{- slot->params.input_prefix = data["input_prefix"];+ const auto & prompt = data.find("prompt");+ if (prompt == data.end()) {+ send_error(task, "Either \"prompt\" or \"messages\" must be provided", ERROR_TYPE_INVALID_REQUEST);+ return false;+ } else {+ slot.prompt = *prompt;+ }+ if (slot.prompt.is_array() && slot.prompt.size() == 0) {+ send_error(task, "\"prompt\" cannot be an empty array", ERROR_TYPE_INVALID_REQUEST);+ return false;+ }}- else++ // penalize user-provided tokens{- slot->params.input_prefix = "";- }+ slot.sparams.penalty_prompt_tokens.clear();+ slot.sparams.use_penalty_prompt_tokens = false;- if (data.count("input_suffix") != 0)- {- slot->params.input_suffix = data["input_suffix"];- }- else- {- slot->params.input_suffix = "";- }+ const auto & penalty_prompt = data.find("penalty_prompt");- if (data.count("prompt") != 0)- {- slot->prompt = data["prompt"];- }- else- {- slot->prompt = "";- }+ if (penalty_prompt != data.end()) {+ if (penalty_prompt->is_string()) {+ const auto penalty_prompt_string = penalty_prompt->get<std::string>();+ slot.sparams.penalty_prompt_tokens = llama_tokenize(model, penalty_prompt_string, false);- slot->sparams.penalty_prompt_tokens.clear();- slot->sparams.use_penalty_prompt_tokens = false;- const auto &penalty_prompt = data.find("penalty_prompt");- if (penalty_prompt != data.end())- {- if (penalty_prompt->is_string())- {- const auto penalty_prompt_string = penalty_prompt->get<std::string>();- auto penalty_tokens = llama_tokenize(model, penalty_prompt_string, false);- slot->sparams.penalty_prompt_tokens.swap(penalty_tokens);- if (slot->params.n_predict > 0)- {- slot->sparams.penalty_prompt_tokens.reserve(slot->sparams.penalty_prompt_tokens.size() + slot->params.n_predict);+ if (slot.params.n_predict > 0) {+ slot.sparams.penalty_prompt_tokens.reserve(slot.sparams.penalty_prompt_tokens.size() + slot.params.n_predict);+ }+ slot.sparams.use_penalty_prompt_tokens = true;++ LOG_VERBOSE("penalty_prompt_tokens", {+ {"id_slot", slot.id},+ {"tokens", slot.sparams.penalty_prompt_tokens},+ });}- slot->sparams.use_penalty_prompt_tokens = true;- }- else if (penalty_prompt->is_array())- {- const auto n_tokens = penalty_prompt->size();- slot->sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot->params.n_predict));- const int n_vocab = llama_n_vocab(model);- for (const auto &penalty_token : *penalty_prompt)- {- if (penalty_token.is_number_integer())- {- const auto tok = penalty_token.get<llama_token>();- if (tok >= 0 && tok < n_vocab)- {- slot->sparams.penalty_prompt_tokens.push_back(tok);+ else if (penalty_prompt->is_array()) {+ const auto n_tokens = penalty_prompt->size();+ slot.sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot.params.n_predict));++ const int n_vocab = llama_n_vocab(model);+ for (const auto & penalty_token : *penalty_prompt) {+ if (penalty_token.is_number_integer()) {+ const auto tok = penalty_token.get<llama_token>();+ if (tok >= 0 && tok < n_vocab) {+ slot.sparams.penalty_prompt_tokens.push_back(tok);+ }}}+ slot.sparams.use_penalty_prompt_tokens = true;++ LOG_VERBOSE("penalty_prompt_tokens", {+ {"id_slot", slot.id},+ {"tokens", slot.sparams.penalty_prompt_tokens},+ });}- slot->sparams.use_penalty_prompt_tokens = true;}}- slot->sparams.logit_bias.clear();-- if (json_value(data, "ignore_eos", false)){- slot->sparams.logit_bias[llama_token_eos(model)] = -INFINITY;- }+ slot.sparams.logit_bias.clear();- const auto &logit_bias = data.find("logit_bias");- if (logit_bias != data.end() && logit_bias->is_array())- {- const int n_vocab = llama_n_vocab(model);- for (const auto &el : *logit_bias)- {- if (el.is_array() && el.size() == 2)- {- float bias;- if (el[1].is_number())- {- bias = el[1].get<float>();- }- else if (el[1].is_boolean() && !el[1].get<bool>())- {- bias = -INFINITY;- }- else- {- continue;- }+ if (json_value(data, "ignore_eos", false)) {+ slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;+ }- if (el[0].is_number_integer())- {- llama_token tok = el[0].get<llama_token>();- if (tok >= 0 && tok < n_vocab)- {- slot->sparams.logit_bias[tok] = bias;+ const auto & logit_bias = data.find("logit_bias");+ if (logit_bias != data.end() && logit_bias->is_array()) {+ const int n_vocab = llama_n_vocab(model);+ for (const auto & el : *logit_bias) {+ // TODO: we may want to throw errors here, in case "el" is incorrect+ if (el.is_array() && el.size() == 2) {+ float bias;+ if (el[1].is_number()) {+ bias = el[1].get<float>();+ } else if (el[1].is_boolean() && !el[1].get<bool>()) {+ bias = -INFINITY;+ } else {+ continue;}- }- else if (el[0].is_string())- {- auto toks = llama_tokenize(model, el[0].get<std::string>(), false);- for (auto tok : toks)- {- slot->sparams.logit_bias[tok] = bias;++ if (el[0].is_number_integer()) {+ llama_token tok = el[0].get<llama_token>();+ if (tok >= 0 && tok < n_vocab) {+ slot.sparams.logit_bias[tok] = bias;+ }+ } else if (el[0].is_string()) {+ auto toks = llama_tokenize(model, el[0].get<std::string>(), false);+ for (auto tok : toks) {+ slot.sparams.logit_bias[tok] = bias;+ }}}}}}- slot->params.antiprompt.clear();-- const auto &stop = data.find("stop");- if (stop != data.end() && stop->is_array()){- for (const auto &word : *stop)- {- if (!word.empty())- {- slot->params.antiprompt.push_back(word);- }- }- }+ slot.params.antiprompt.clear();- const auto &samplers_sequence = data.find("samplers");- if (samplers_sequence != data.end() && samplers_sequence->is_array())- {- std::vector<std::string> sampler_names;- for (const auto &sampler_name : *samplers_sequence)- {- if (sampler_name.is_string())- {- sampler_names.emplace_back(sampler_name);+ const auto & stop = data.find("stop");+ if (stop != data.end() && stop->is_array()) {+ for (const auto & word : *stop) {+ if (!word.empty()) {+ slot.params.antiprompt.push_back(word);+ }}}- slot->sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);- }- else- {- slot->sparams.samplers_sequence = default_sparams.samplers_sequence;}- if (multimodal){- const auto &images_data = data.find("image_data");- if (images_data != data.end() && images_data->is_array())- {- for (const auto &img : *images_data)- {- const std::vector<uint8_t> image_buffer = base64_decode(img["data"].get<std::string>());-- slot_image img_sl;- img_sl.id = img.count("id") != 0 ? img["id"].get<int>() : slot->images.size();- img_sl.img_data = clip_image_u8_init();- if (!clip_image_load_from_bytes(image_buffer.data(), image_buffer.size(), img_sl.img_data))- {- LOG_ERROR("failed to load image", {- {"slot_id", slot->id},- {"img_sl_id", img_sl.id}- });- return false;- }- LOG_VERBOSE("image loaded", {- {"slot_id", slot->id},- {"img_sl_id", img_sl.id}- });- img_sl.request_encode_image = true;- slot->images.push_back(img_sl);- }- // process prompt- // example: system prompt [img-102] user [img-103] describe [img-134] -> [{id: 102, prefix: 'system prompt '}, {id: 103, prefix: ' user '}, {id: 134, prefix: ' describe '}]}- if (slot->images.size() > 0 && !slot->prompt.is_array())- {- std::string prompt = slot->prompt.get<std::string>();- size_t pos = 0, begin_prefix = 0;- std::string pattern = "[img-";- while ((pos = prompt.find(pattern, pos)) != std::string::npos) {- size_t end_prefix = pos;- pos += pattern.length();- size_t end_pos = prompt.find(']', pos);- if (end_pos != std::string::npos)- {- std::string image_id = prompt.substr(pos, end_pos - pos);- try- {- int img_id = std::stoi(image_id);- bool found = false;- for (slot_image &img : slot->images)- {- if (img.id == img_id) {- found = true;- img.prefix_prompt = prompt.substr(begin_prefix, end_prefix - begin_prefix);- begin_prefix = end_pos + 1;- break;- }- }- if (!found) {- LOG_TEE("ERROR: Image with id: %i, not found.\n", img_id);- slot->images.clear();- return false;- }- } catch (const std::invalid_argument& e) {- LOG_TEE("Invalid image number id in prompt\n");- slot->images.clear();- return false;- }- }+ const auto & samplers_sequence = data.find("samplers");+ if (samplers_sequence != data.end() && samplers_sequence->is_array()) {+ std::vector<std::string> sampler_names;+ for (const auto & sampler_name : *samplers_sequence) {+ if (sampler_name.is_string()) {+ sampler_names.emplace_back(sampler_name);}- slot->prompt = "";- slot->params.input_suffix = prompt.substr(begin_prefix);- slot->params.cache_prompt = false; // multimodal doesn't support cache prompt}+ slot.sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);+ } else {+ slot.sparams.samplers_sequence = default_sparams.samplers_sequence;}}- if (slot->ctx_sampling != nullptr){- llama_sampling_free(slot->ctx_sampling);+ if (slot.ctx_sampling != nullptr) {+ llama_sampling_free(slot.ctx_sampling);+ }+ slot.ctx_sampling = llama_sampling_init(slot.sparams);+ if (slot.ctx_sampling == nullptr) {+ // for now, the only error that may happen here is invalid grammar+ send_error(task, "Failed to parse grammar", ERROR_TYPE_INVALID_REQUEST);+ return false;+ }+ llama_set_rng_seed(ctx, slot.params.seed);}- slot->ctx_sampling = llama_sampling_init(slot->sparams);- llama_set_rng_seed(ctx, slot->params.seed);- slot->command = LOAD_PROMPT;- all_slots_are_idle = false;+ slot.command = SLOT_COMMAND_LOAD_PROMPT;+ slot.prompt_tokens.clear();LOG_INFO("slot is processing task", {- {"slot_id", slot->id},- {"task_id", slot->task_id},+ {"id_slot", slot.id},+ {"id_task", slot.id_task},});return true;}void kv_cache_clear() {+ LOG_VERBOSE("clearing KV cache", {});+// clear the entire KV cachellama_kv_cache_clear(ctx);clean_kv_cache = false;}- void update_system_prompt() {+ void system_prompt_update() {+ LOG_VERBOSE("system prompt update", {+ {"system_prompt", system_prompt},+ });+kv_cache_clear();system_tokens.clear();@@ -891,14 +1048,14 @@ struct llama_server_contextllama_batch_clear(batch);- for (int i = 0; i < (int)system_tokens.size(); ++i)- {+ for (int i = 0; i < (int)system_tokens.size(); ++i) {llama_batch_add(batch, system_tokens[i], i, { 0 }, false);}- for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch)- {- const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i));+ const int32_t n_batch = llama_n_batch(ctx);++ for (int32_t i = 0; i < batch.n_tokens; i += n_batch) {+ const int32_t n_tokens = std::min(params.n_batch, batch.n_tokens - i);llama_batch batch_view = {n_tokens,batch.token + i,@@ -909,78 +1066,42 @@ struct llama_server_contextbatch.logits + i,0, 0, 0, // unused};- if (llama_decode(ctx, batch_view) != 0)- {++ if (llama_decode(ctx, batch_view) != 0) {LOG_TEE("%s: llama_decode() failed\n", __func__);return;}}// assign the system KV cache to all parallel sequences- for (int32_t i = 1; i < params.n_parallel; ++i)- {- llama_kv_cache_seq_cp(ctx, 0, i, 0, system_tokens.size());+ for (int32_t i = 1; i <= params.n_parallel; ++i) {+ llama_kv_cache_seq_cp(ctx, 0, i, -1, -1);}}- LOG_TEE("system prompt updated\n");system_need_update = false;}- void notify_system_prompt_changed() {- // release all slots- for (llama_client_slot &slot : slots)- {- slot.release();- }-- system_need_update = true;- }-- void process_system_prompt_data(const json &sys_props) {+ void system_prompt_set(const json & sys_props) {system_prompt = sys_props.value("prompt", "");name_user = sys_props.value("anti_prompt", "");name_assistant = sys_props.value("assistant_name", "");+ LOG_VERBOSE("system prompt process", {+ {"system_prompt", system_prompt},+ {"name_user", name_user},+ {"name_assistant", name_assistant},+ });- notify_system_prompt_changed();- }-- static size_t find_stopping_strings(const std::string &text, const size_t last_token_size,- const stop_type type, llama_client_slot &slot)- {- size_t stop_pos = std::string::npos;-- for (const std::string &word : slot.params.antiprompt)- {- size_t pos;- if (type == STOP_FULL)- {- const size_t tmp = word.size() + last_token_size;- const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;- pos = text.find(word, from_pos);- }- else- {- pos = find_partial_stop_string(word, text);- }- if (pos != std::string::npos &&- (stop_pos == std::string::npos || pos < stop_pos))- {- if (type == STOP_FULL)- {- slot.stopped_word = true;- slot.stopping_word = word;- slot.has_next_token = false;- }- stop_pos = pos;- }+ // release all slots+ for (server_slot & slot : slots) {+ slot.release();}- return stop_pos;+ system_need_update = true;}- bool process_token(completion_token_output &result, llama_client_slot &slot) {+ bool process_token(completion_token_output & result, server_slot & slot) {// remember which tokens were sampled - used for repetition penalties during samplingconst std::string token_str = llama_token_to_piece(ctx, result.tok);slot.sampled = result.tok;@@ -989,34 +1110,26 @@ struct llama_server_contextslot.generated_text += token_str;slot.has_next_token = true;- if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1)- {+ if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1) {// we can change penalty_prompt_tokens because it is always created from scratch each requestslot.ctx_sampling->params.penalty_prompt_tokens.push_back(result.tok);}// check if there is incomplete UTF-8 character at the endbool incomplete = false;- for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i)- {+ for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i) {unsigned char c = slot.generated_text[slot.generated_text.size() - i];- if ((c & 0xC0) == 0x80)- {+ if ((c & 0xC0) == 0x80) {// continuation byte: 10xxxxxxcontinue;}- if ((c & 0xE0) == 0xC0)- {+ if ((c & 0xE0) == 0xC0) {// 2-byte character: 110xxxxx ...incomplete = i < 2;- }- else if ((c & 0xF0) == 0xE0)- {+ } else if ((c & 0xF0) == 0xE0) {// 3-byte character: 1110xxxx ...incomplete = i < 3;- }- else if ((c & 0xF8) == 0xF0)- {+ } else if ((c & 0xF8) == 0xF0) {// 4-byte character: 11110xxx ...incomplete = i < 4;}@@ -1024,210 +1137,196 @@ struct llama_server_contextbreak;}- if (!incomplete)- {- size_t pos = std::min(slot.sent_count, slot.generated_text.size());+ if (!incomplete) {+ size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());+const std::string str_test = slot.generated_text.substr(pos);bool is_stop_full = false;- size_t stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_FULL, slot);- if (stop_pos != std::string::npos)- {++ size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_FULL);+ if (stop_pos != std::string::npos) {is_stop_full = true;slot.generated_text.erase(slot.generated_text.begin() + pos + stop_pos,slot.generated_text.end());- pos = std::min(slot.sent_count, slot.generated_text.size());- }- else- {+ pos = std::min(slot.n_sent_text, slot.generated_text.size());+ } else {is_stop_full = false;- stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_PARTIAL, slot);+ stop_pos = slot.find_stopping_strings(str_test, token_str.size(), STOP_TYPE_PARTIAL);}// check if there is any token to predict- if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0))- {+ if (stop_pos == std::string::npos || (!slot.has_next_token && !is_stop_full && stop_pos > 0)) {// no send the stop word in the responseresult.text_to_send = slot.generated_text.substr(pos, std::string::npos);- slot.sent_count += result.text_to_send.size();+ slot.n_sent_text += result.text_to_send.size();// add the token to slot queue and cache}+slot.add_token_string(result);- if (slot.params.stream)- {+ if (slot.params.stream) {send_partial_response(slot, result);}}- if (incomplete)- {+ if (incomplete) {slot.has_next_token = true;}// check the limits- if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params))- {- slot.stopped_limit = true;+ if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params)) {+ slot.stopped_limit = true;slot.has_next_token = false;++ LOG_VERBOSE("stopped by limit", {+ {"id_slot", slot.id},+ {"id_task", slot.id_task},+ {"n_decoded", slot.n_decoded},+ {"n_predict", slot.params.n_predict},+ });}- if (!slot.cache_tokens.empty() && result.tok == llama_token_eos(model))- {- slot.stopped_eos = true;+ if (result.tok == llama_token_eos(model)) {+ slot.stopped_eos = true;slot.has_next_token = false;+LOG_VERBOSE("eos token found", {});}LOG_VERBOSE("next token", {- {"token", result.tok},- {"token_text", tokens_to_output_formatted_string(ctx, result.tok)},- {"has_next_token", slot.has_next_token},- {"n_remain", slot.n_remaining},- {"num_tokens_predicted", slot.n_decoded},- {"stopped_eos", slot.stopped_eos},- {"stopped_word", slot.stopped_word},- {"stopped_limit", slot.stopped_limit},- {"stopping_word", slot.stopping_word},- });+ {"id_slot", slot.id},+ {"id_task", slot.id_task},+ {"token", result.tok},+ {"token_text", tokens_to_output_formatted_string(ctx, result.tok)},+ {"has_next_token", slot.has_next_token},+ {"n_remain", slot.n_remaining},+ {"n_decoded", slot.n_decoded},+ {"stopped_eos", slot.stopped_eos},+ {"stopped_word", slot.stopped_word},+ {"stopped_limit", slot.stopped_limit},+ {"stopping_word", slot.stopping_word},+ });return slot.has_next_token; // continue}- bool process_images(llama_client_slot &slot) const- {- for (slot_image &img : slot.images)- {- if (!img.request_encode_image)- {- continue;- }-- if (!llava_image_embed_make_with_clip_img(clp_ctx, params.n_threads, img.img_data, &img.image_embedding, &img.image_tokens)) {- LOG_TEE("Error processing the given image");- return false;- }--- img.request_encode_image = false;- }-- return slot.images.size() > 0;- }-- void send_error(task_server& task, const std::string &error)- {- LOG_TEE("task %i - error: %s\n", task.id, error.c_str());- task_result res;- res.id = task.id;- res.multitask_id = task.multitask_id;- res.stop = false;- res.error = true;- res.result_json = { { "content", error } };- queue_results.send(res);- }-- json get_formated_generation(llama_client_slot &slot)- {+ json get_formated_generation(const server_slot & slot) const {const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));- const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() &&- eos_bias->second < 0.0f && std::isinf(eos_bias->second);+ const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && eos_bias->second < 0.0f && std::isinf(eos_bias->second);+std::vector<std::string> samplers_sequence;- for (const auto &sampler_type : slot.sparams.samplers_sequence)- {+ samplers_sequence.reserve(slot.sparams.samplers_sequence.size());+ for (const auto & sampler_type : slot.sparams.samplers_sequence) {samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type));}return json {- {"n_ctx", slot.n_ctx},- {"n_predict", slot.n_predict},- {"model", params.model_alias},- {"seed", slot.params.seed},- {"temperature", slot.sparams.temp},- {"dynatemp_range", slot.sparams.dynatemp_range},- {"dynatemp_exponent", slot.sparams.dynatemp_exponent},- {"top_k", slot.sparams.top_k},- {"top_p", slot.sparams.top_p},- {"min_p", slot.sparams.min_p},- {"tfs_z", slot.sparams.tfs_z},- {"typical_p", slot.sparams.typical_p},- {"repeat_last_n", slot.sparams.penalty_last_n},- {"repeat_penalty", slot.sparams.penalty_repeat},- {"presence_penalty", slot.sparams.penalty_present},- {"frequency_penalty", slot.sparams.penalty_freq},- {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},+ {"n_ctx", slot.n_ctx},+ {"n_predict", slot.n_predict},+ {"model", params.model_alias},+ {"seed", slot.params.seed},+ {"temperature", slot.sparams.temp},+ {"dynatemp_range", slot.sparams.dynatemp_range},+ {"dynatemp_exponent", slot.sparams.dynatemp_exponent},+ {"top_k", slot.sparams.top_k},+ {"top_p", slot.sparams.top_p},+ {"min_p", slot.sparams.min_p},+ {"tfs_z", slot.sparams.tfs_z},+ {"typical_p", slot.sparams.typical_p},+ {"repeat_last_n", slot.sparams.penalty_last_n},+ {"repeat_penalty", slot.sparams.penalty_repeat},+ {"presence_penalty", slot.sparams.penalty_present},+ {"frequency_penalty", slot.sparams.penalty_freq},+ {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},{"use_penalty_prompt_tokens", slot.sparams.use_penalty_prompt_tokens},- {"mirostat", slot.sparams.mirostat},- {"mirostat_tau", slot.sparams.mirostat_tau},- {"mirostat_eta", slot.sparams.mirostat_eta},- {"penalize_nl", slot.sparams.penalize_nl},- {"stop", slot.params.antiprompt},- {"n_predict", slot.params.n_predict},- {"n_keep", params.n_keep},- {"ignore_eos", ignore_eos},- {"stream", slot.params.stream},- {"logit_bias", slot.sparams.logit_bias},- {"n_probs", slot.sparams.n_probs},- {"min_keep", slot.sparams.min_keep},- {"grammar", slot.sparams.grammar},- {"samplers", samplers_sequence}+ {"mirostat", slot.sparams.mirostat},+ {"mirostat_tau", slot.sparams.mirostat_tau},+ {"mirostat_eta", slot.sparams.mirostat_eta},+ {"penalize_nl", slot.sparams.penalize_nl},+ {"stop", slot.params.antiprompt},+ {"n_predict", slot.params.n_predict}, // TODO: fix duplicate key n_predict+ {"n_keep", params.n_keep},+ {"ignore_eos", ignore_eos},+ {"stream", slot.params.stream},+ {"logit_bias", slot.sparams.logit_bias},+ {"n_probs", slot.sparams.n_probs},+ {"min_keep", slot.sparams.min_keep},+ {"grammar", slot.sparams.grammar},+ {"samplers", samplers_sequence}};}- void send_partial_response(llama_client_slot &slot, completion_token_output tkn)- {- task_result res;- res.id = slot.task_id;- res.multitask_id = slot.multitask_id;- res.error = false;- res.stop = false;+ void send_error(const server_task & task, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {+ send_error(task.id, task.id_multi, error, type);+ }- res.result_json = json- {+ void send_error(const server_slot & slot, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {+ send_error(slot.id_task, slot.id_multi, error, type);+ }++ void send_error(const int id_task, const int id_multi, const std::string & error, const enum error_type type = ERROR_TYPE_SERVER) {+ LOG_TEE("task %i - error: %s\n", id_task, error.c_str());++ server_task_result res;+ res.id = id_task;+ res.id_multi = id_multi;+ res.stop = false;+ res.error = true;+ res.data = format_error_response(error, type);++ queue_results.send(res);+ }++ void send_partial_response(server_slot & slot, completion_token_output tkn) {+ server_task_result res;+ res.id = slot.id_task;+ res.id_multi = slot.id_multi;+ res.error = false;+ res.stop = false;+ res.data = json {{"content", tkn.text_to_send},{"stop", false},- {"slot_id", slot.id},- {"multimodal", multimodal}+ {"id_slot", slot.id},+ {"multimodal", false}};- if (slot.sparams.n_probs > 0)- {- std::vector<completion_token_output> probs_output = {};+ if (slot.sparams.n_probs > 0) {const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);- size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());- size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size());- if (probs_pos < probs_stop_pos)- {- probs_output = std::vector<completion_token_output>(slot.generated_token_probs.begin() + probs_pos, slot.generated_token_probs.begin() + probs_stop_pos);+ const size_t probs_pos = std::min(slot.n_sent_token_probs, slot.generated_token_probs.size());+ const size_t probs_stop_pos = std::min(slot.n_sent_token_probs + to_send_toks.size(), slot.generated_token_probs.size());++ std::vector<completion_token_output> probs_output;+ if (probs_pos < probs_stop_pos) {+ probs_output = std::vector<completion_token_output>(+ slot.generated_token_probs.begin() + probs_pos,+ slot.generated_token_probs.begin() + probs_stop_pos);}- slot.sent_token_probs_index = probs_stop_pos;- res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs_output);+ slot.n_sent_token_probs = probs_stop_pos;++ res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs_output);}- if (slot.oaicompat)- {- res.result_json["oaicompat_token_ctr"] = slot.n_decoded;- res.result_json["model"] = slot.oaicompat_model;+ if (slot.oaicompat) {+ res.data["oaicompat_token_ctr"] = slot.n_decoded;+ res.data["model"] = slot.oaicompat_model;}queue_results.send(res);}- void send_final_response(llama_client_slot &slot)- {- task_result res;- res.id = slot.task_id;- res.multitask_id = slot.multitask_id;- res.error = false;- res.stop = true;-- res.result_json = json- {+ void send_final_response(const server_slot & slot) {+ server_task_result res;+ res.id = slot.id_task;+ res.id_multi = slot.id_multi;+ res.error = false;+ res.stop = true;+ res.data = json {{"content", !slot.params.stream ? slot.generated_text : ""},- {"slot_id", slot.id},+ {"id_slot", slot.id},{"stop", true},{"model", params.model_alias},{"tokens_predicted", slot.n_decoded},- {"tokens_evaluated", slot.num_prompt_tokens},+ {"tokens_evaluated", slot.n_prompt_tokens},{"generation_settings", get_formated_generation(slot)},{"prompt", slot.prompt},{"truncated", slot.truncated},@@ -1239,80 +1338,91 @@ struct llama_server_context{"timings", slot.get_formated_timings()}};- if (slot.sparams.n_probs > 0)- {- std::vector<completion_token_output> probs = {};- if (!slot.params.stream && slot.stopped_word)- {+ if (slot.sparams.n_probs > 0) {+ std::vector<completion_token_output> probs;+ if (!slot.params.stream && slot.stopped_word) {const std::vector<llama_token> stop_word_toks = llama_tokenize(ctx, slot.stopping_word, false);- probs = std::vector<completion_token_output>(slot.generated_token_probs.begin(), slot.generated_token_probs.end() - stop_word_toks.size());- }- else- {++ probs = std::vector<completion_token_output>(+ slot.generated_token_probs.begin(),+ slot.generated_token_probs.end() - stop_word_toks.size());+ } else {probs = std::vector<completion_token_output>(- slot.generated_token_probs.begin(),- slot.generated_token_probs.end());+ slot.generated_token_probs.begin(),+ slot.generated_token_probs.end());}- res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);++ res.data["completion_probabilities"] = probs_vector_to_json(ctx, probs);}- if (slot.oaicompat)- {- res.result_json["oaicompat_token_ctr"] = slot.n_decoded;- res.result_json["model"] = slot.oaicompat_model;+ if (slot.oaicompat) {+ res.data["oaicompat_token_ctr"] = slot.n_decoded;+ res.data["model"] = slot.oaicompat_model;}queue_results.send(res);}- void send_embedding(llama_client_slot &slot)- {- task_result res;- res.id = slot.task_id;- res.multitask_id = slot.multitask_id;- res.error = false;- res.stop = true;+ void send_embedding(const server_slot & slot, const llama_batch & batch) {+ server_task_result res;+ res.id = slot.id_task;+ res.id_multi = slot.id_multi;+ res.error = false;+ res.stop = true;const int n_embd = llama_n_embd(model);- if (!params.embedding)- {- LOG_WARNING("embedding disabled", {- {"params.embedding", params.embedding},- });- res.result_json = json- {- {"embedding", std::vector<float>(n_embd, 0.0f)},- };- }- else- {- const float *data = llama_get_embeddings(ctx);- std::vector<float> embedding(data, data + n_embd);- res.result_json = json- {- {"embedding", embedding },++ std::vector<float> embd_res(n_embd, 0.0f);++ for (int i = 0; i < batch.n_tokens; ++i) {+ if (!batch.logits[i] || batch.seq_id[i][0] != slot.id + 1) {+ continue;+ }++ const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]);+ if (embd == NULL) {+ embd = llama_get_embeddings_ith(ctx, i);+ }++ if (embd == NULL) {+ LOG_ERROR("failed to get embeddings", {+ {"token", batch.token [i]},+ {"seq_id", batch.seq_id[i][0]}+ });++ res.data = json {+ {"embedding", std::vector<float>(n_embd, 0.0f)},+ };++ continue;+ }++ llama_embd_normalize(embd, embd_res.data(), n_embd);++ res.data = json {+ {"embedding", embd_res},};}+queue_results.send(res);}- void request_completion(int task_id, json data, bool infill, bool embedding, int multitask_id)- {- task_server task;- task.id = task_id;- task.target_id = 0;- task.data = std::move(data);- task.infill_mode = infill;- task.embedding_mode = embedding;- task.type = TASK_TYPE_COMPLETION;- task.multitask_id = multitask_id;+ void request_completion(int id_task, int id_multi, json data, bool infill, bool embedding) {+ server_task task;+ task.id = id_task;+ task.id_multi = id_multi;+ task.id_target = 0;+ task.data = std::move(data);+ task.infill = infill;+ task.embedding = embedding;+ task.type = SERVER_TASK_TYPE_COMPLETION;// when a completion task's prompt array is not a singleton, we split it into multiple requests// otherwise, it's a single-prompt task, we actually queue it// if there's numbers in the prompt array it will be treated as an array of tokensif (task.data.count("prompt") != 0 && task.data.at("prompt").size() > 1) {bool numbers = false;- for (const auto& e : task.data.at("prompt")) {+ for (const auto & e : task.data.at("prompt")) {if (e.is_number()) {numbers = true;break;@@ -1327,96 +1437,23 @@ struct llama_server_contextif (numbers) {queue_tasks.post(task);} else {- split_multiprompt_task(task_id, task);+ split_multiprompt_task(id_task, task);}} else {- // an empty prompt can make slot become buggy- if (task.data.contains("prompt") && task.data["prompt"].is_string() && task.data["prompt"].get<std::string>().empty()) {- task.data["prompt"] = " "; // add a space so that we have one token- }queue_tasks.post(task);}}- // for multiple images processing- bool ingest_images(llama_client_slot &slot, int n_batch)- {- int image_idx = 0;-- while (image_idx < (int) slot.images.size())- {- slot_image &img = slot.images[image_idx];-- // process prefix prompt- for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)- {- const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));- llama_batch batch_view = {- n_tokens,- batch.token + i,- nullptr,- batch.pos + i,- batch.n_seq_id + i,- batch.seq_id + i,- batch.logits + i,- 0, 0, 0, // unused- };- if (llama_decode(ctx, batch_view))- {- LOG_TEE("%s : failed to eval\n", __func__);- return false;- }- }-- // process image with llm- for (int i = 0; i < img.image_tokens; i += n_batch)- {- int n_eval = img.image_tokens - i;- if (n_eval > n_batch)- {- n_eval = n_batch;- }-- const int n_embd = llama_n_embd(model);- llama_batch batch_img = { n_eval, nullptr, (img.image_embedding + i * n_embd), nullptr, nullptr, nullptr, nullptr, slot.n_past, 1, 0, };- if (llama_decode(ctx, batch_img))- {- LOG_TEE("%s : failed to eval image\n", __func__);- return false;- }- slot.n_past += n_eval;- }- image_idx++;-- llama_batch_clear(batch);-- // append prefix of next image- const auto json_prompt = (image_idx >= (int) slot.images.size()) ?- slot.params.input_suffix : // no more images, then process suffix prompt- (json)(slot.images[image_idx].prefix_prompt);+ void request_cancel(int id_task) {+ server_task task;+ task.type = SERVER_TASK_TYPE_CANCEL;+ task.id_target = id_task;- std::vector<llama_token> append_tokens = tokenize(json_prompt, false); // has next image- for (int i = 0; i < (int) append_tokens.size(); ++i)- {- llama_batch_add(batch, append_tokens[i], system_tokens.size() + slot.n_past, { slot.id }, true);- slot.n_past += 1;- }- }-- return true;- }-- void request_cancel(int task_id)- {- task_server task;- task.type = TASK_TYPE_CANCEL;- task.target_id = task_id;queue_tasks.post(task);}- void split_multiprompt_task(int multitask_id, task_server& multiprompt_task)- {- int prompt_count = multiprompt_task.data.at("prompt").size();+ void split_multiprompt_task(int id_multi, const server_task & multiprompt_task) {+ const int prompt_count = multiprompt_task.data.at("prompt").size();if (prompt_count <= 1) {send_error(multiprompt_task, "error while handling multiple prompts");return;@@ -1424,208 +1461,239 @@ struct llama_server_context// generate all the ID for subtaskstd::vector<int> subtask_ids(prompt_count);- for (int i = 0; i < prompt_count; i++)- {+ for (int i = 0; i < prompt_count; i++) {subtask_ids[i] = queue_tasks.get_new_id();}// queue up the multitask so we can track its subtask progression- queue_tasks.add_multitask(multitask_id, subtask_ids);+ queue_tasks.add_multitask(id_multi, subtask_ids);// add subtasks- for (int i = 0; i < prompt_count; i++)- {+ for (int i = 0; i < prompt_count; i++) {json subtask_data = multiprompt_task.data;subtask_data["prompt"] = subtask_data["prompt"][i];// subtasks inherit everything else (infill mode, embedding mode, etc.)- request_completion(subtask_ids[i], subtask_data, multiprompt_task.infill_mode, multiprompt_task.embedding_mode, multitask_id);+ request_completion(subtask_ids[i], id_multi, subtask_data, multiprompt_task.infill, multiprompt_task.embedding);}}- void process_single_task(task_server& task)- {- switch (task.type)- {- case TASK_TYPE_COMPLETION: {- llama_client_slot *slot = get_slot(json_value(task.data, "slot_id", -1));- if (slot == nullptr)- {- // if no slot is available, we defer this task for processing later- LOG_VERBOSE("no slot is available", {{"task_id", task.id}});- queue_tasks.defer(task);- break;- }-- if (task.data.contains("system_prompt"))+ void process_single_task(const server_task & task) {+ switch (task.type) {+ case SERVER_TASK_TYPE_COMPLETION:{- if (!all_slots_are_idle) {- send_error(task, "system prompt can only be updated when all slots are idle");+ server_slot * slot = get_slot(json_value(task.data, "id_slot", -1));+ if (slot == nullptr) {+ // if no slot is available, we defer this task for processing later+ LOG_VERBOSE("no slot is available", {{"id_task", task.id}});+ queue_tasks.defer(task);break;}- process_system_prompt_data(task.data["system_prompt"]);- // reset cache_tokens for all slots- for (llama_client_slot &slot : slots)- {- slot.cache_tokens.clear();- slot.n_past = 0;- slot.n_past_se = 0;+ if (task.data.contains("system_prompt")) {+ system_prompt_set(task.data["system_prompt"]);++ for (server_slot & slot : slots) {+ slot.n_past = 0;+ slot.n_past_se = 0;+ }}- }- slot->reset();+ slot->reset();- slot->infill = task.infill_mode;- slot->embedding = task.embedding_mode;- slot->task_id = task.id;- slot->multitask_id = task.multitask_id;+ slot->id_task = task.id;+ slot->id_multi = task.id_multi;+ slot->infill = task.infill;+ slot->embedding = task.embedding;- if (!launch_slot_with_data(slot, task.data))+ if (!launch_slot_with_task(*slot, task)) {+ LOG_ERROR("error while launching slot", task.data);+ break;+ }+ } break;+ case SERVER_TASK_TYPE_CANCEL:{- // send error result- send_error(task, "internal_error");- break;- }- } break;- case TASK_TYPE_CANCEL: { // release slot linked with the task id- for (auto & slot : slots)- {- if (slot.task_id == task.target_id)- {- slot.release();- break;+ // release slot linked with the task id+ for (auto & slot : slots) {+ if (slot.id_task == task.id_target) {+ slot.release();+ break;+ }}- }- } break;- case TASK_TYPE_NEXT_RESPONSE: {- // do nothing- } break;- case TASK_TYPE_METRICS: {- json slots_data = json::array();- int n_idle_slots = 0;- int n_processing_slots = 0;-- for (llama_client_slot &slot: slots) {- json slot_data = get_formated_generation(slot);- slot_data["id"] = slot.id;- slot_data["task_id"] = slot.task_id;- slot_data["state"] = slot.state;- slot_data["prompt"] = slot.prompt;- slot_data["next_token"] = {+ } break;+ case SERVER_TASK_TYPE_NEXT_RESPONSE:+ {+ // do nothing+ } break;+ case SERVER_TASK_TYPE_METRICS:+ {+ json slots_data = json::array();++ int n_idle_slots = 0;+ int n_processing_slots = 0;++ for (server_slot & slot : slots) {+ json slot_data = get_formated_generation(slot);+ slot_data["id"] = slot.id;+ slot_data["id_task"] = slot.id_task;+ slot_data["state"] = slot.state;+ slot_data["prompt"] = slot.prompt;+ slot_data["next_token"] = {{"has_next_token", slot.has_next_token},- {"n_remain", slot.n_remaining},- {"num_tokens_predicted", slot.n_decoded},- {"stopped_eos", slot.stopped_eos},- {"stopped_word", slot.stopped_word},- {"stopped_limit", slot.stopped_limit},- {"stopping_word", slot.stopping_word},- };- if (slot_data["state"] == IDLE) {- n_idle_slots++;- } else {- n_processing_slots++;+ {"n_remain", slot.n_remaining},+ {"n_decoded", slot.n_decoded},+ {"stopped_eos", slot.stopped_eos},+ {"stopped_word", slot.stopped_word},+ {"stopped_limit", slot.stopped_limit},+ {"stopping_word", slot.stopping_word},+ };++ if (slot_data["state"] == SLOT_STATE_IDLE) {+ n_idle_slots++;+ } else {+ n_processing_slots++;+ }++ slots_data.push_back(slot_data);}- slots_data.push_back(slot_data);- }- LOG_INFO("slot data", {- {"task_id", task.id},- {"n_idle_slots", n_idle_slots},- {"n_processing_slots", n_processing_slots}- });- LOG_VERBOSE("slot data", {- {"task_id", task.id},- {"n_idle_slots", n_idle_slots},- {"n_processing_slots", n_processing_slots},- {"slots", slots_data}- });- task_result res;- res.id = task.id;- res.multitask_id = task.multitask_id;- res.stop = true;- res.error = false;- res.result_json = {+ LOG_INFO("slot data", {+ {"id_task", task.id},+ {"n_idle_slots", n_idle_slots},+ {"n_processing_slots", n_processing_slots}+ });++ LOG_VERBOSE("slot data", {+ {"id_task", task.id},+ {"n_idle_slots", n_idle_slots},+ {"n_processing_slots", n_processing_slots},+ {"slots", slots_data}+ });++ server_task_result res;+ res.id = task.id;+ res.id_multi = task.id_multi;+ res.stop = true;+ res.error = false;+ res.data = {{ "idle", n_idle_slots },{ "processing", n_processing_slots },{ "deferred", queue_tasks.queue_tasks_deferred.size() },+ { "t_start", metrics.t_start},{ "n_prompt_tokens_processed_total", metrics.n_prompt_tokens_processed_total},+ { "t_tokens_generation_total", metrics.t_tokens_generation_total},{ "n_tokens_predicted_total", metrics.n_tokens_predicted_total},+ { "t_prompt_processing_total", metrics.t_prompt_processing_total},{ "n_prompt_tokens_processed", metrics.n_prompt_tokens_processed},{ "t_prompt_processing", metrics.t_prompt_processing},{ "n_tokens_predicted", metrics.n_tokens_predicted},{ "t_tokens_generation", metrics.t_tokens_generation},- { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)},- { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)},+ { "kv_cache_tokens_count", llama_get_kv_cache_token_count(ctx)},+ { "kv_cache_used_cells", llama_get_kv_cache_used_cells(ctx)},- { "slots", slots_data },- };- metrics.reset_bucket();- queue_results.send(res);- } break;+ { "slots", slots_data },+ };++ if (json_value(task.data, "reset_bucket", false)) {+ metrics.reset_bucket();+ }+ queue_results.send(res);+ } break;}}- void on_finish_multitask(task_multi& multitask)- {+ void on_finish_multitask(const server_task_multi & multitask) {// all subtasks done == multitask is done- task_result result;- result.id = multitask.id;- result.stop = true;+ server_task_result result;+ result.id = multitask.id;+ result.stop = true;result.error = false;// collect json results into one json resultstd::vector<json> result_jsons;- for (auto& subres : multitask.results)- {- result_jsons.push_back(subres.result_json);+ for (const auto & subres : multitask.results) {+ result_jsons.push_back(subres.data);result.error = result.error && subres.error;}- result.result_json = json{ { "results", result_jsons } };+ result.data = json {+ { "results", result_jsons }+ };+queue_results.send(result);}- bool update_slots() {- if (system_need_update)- {- LOG_INFO("updating system prompt", {});- update_system_prompt();+ void update_slots() {+ if (system_need_update) {+ system_prompt_update();}- llama_batch_clear(batch);+ // release slots+ for (auto & slot : slots) {+ if (slot.command == SLOT_COMMAND_RELEASE) {+ slot.state = SLOT_STATE_IDLE;+ slot.command = SLOT_COMMAND_NONE;+ slot.t_last_used = ggml_time_us();- if (all_slots_are_idle)- {- if (system_prompt.empty() && clean_kv_cache)- {- LOG_INFO("all slots are idle and system prompt is empty, clear the KV cache", {});- kv_cache_clear();+ LOG_INFO("slot released", {+ {"id_slot", slot.id},+ {"id_task", slot.id_task},+ {"n_ctx", n_ctx},+ {"n_past", slot.n_past},+ {"n_system_tokens", system_tokens.size()},+ {"n_cache_tokens", slot.cache_tokens.size()},+ {"truncated", slot.truncated}+ });++ queue_tasks.notify_slot_changed();}- return true;}- LOG_VERBOSE("posting NEXT_RESPONSE", {});- task_server task;- task.type = TASK_TYPE_NEXT_RESPONSE;- task.target_id = -1;- queue_tasks.post(task);+ // check if all slots are idle+ {+ bool all_idle = true;++ for (auto & slot : slots) {+ if (slot.state != SLOT_STATE_IDLE || slot.command != SLOT_COMMAND_NONE) {+ all_idle = false;+ break;+ }+ }++ if (all_idle) {+ LOG_INFO("all slots are idle", {});+ if (system_prompt.empty() && clean_kv_cache) {+ kv_cache_clear();+ }++ return;+ }+ }- for (llama_client_slot &slot : slots){- if (slot.ga_n == 1)- {- if (slot.is_processing() && system_tokens.size() + slot.cache_tokens.size() >= (size_t) slot.n_ctx)- {+ LOG_VERBOSE("posting NEXT_RESPONSE", {});++ server_task task;+ task.type = SERVER_TASK_TYPE_NEXT_RESPONSE;+ task.id_target = -1;++ queue_tasks.post(task);+ }++ // apply context-shift if needed+ // TODO: simplify and improve+ for (server_slot & slot : slots) {+ if (slot.ga_n == 1) {+ if (slot.is_processing() && (int) system_tokens.size() + slot.n_past >= slot.n_ctx - 1) {// Shift contextconst int n_keep = slot.params.n_keep + add_bos_token;const int n_left = (int) system_tokens.size() + slot.n_past - n_keep;const int n_discard = n_left / 2;LOG_INFO("slot context shift", {- {"slot_id", slot.id},- {"task_id", slot.task_id},+ {"id_slot", slot.id},+ {"id_task", slot.id_task},{"n_keep", n_keep},{"n_left", n_left},{"n_discard", n_discard},@@ -1634,15 +1702,17 @@ struct llama_server_context{"n_system_tokens", system_tokens.size()},{"n_cache_tokens", slot.cache_tokens.size()}});- llama_kv_cache_seq_rm (ctx, slot.id, n_keep , n_keep + n_discard);- llama_kv_cache_seq_add(ctx, slot.id, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);- for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++)- {- slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];- }+ llama_kv_cache_seq_rm (ctx, slot.id + 1, n_keep , n_keep + n_discard);+ llama_kv_cache_seq_add(ctx, slot.id + 1, n_keep + n_discard, system_tokens.size() + slot.n_past, -n_discard);++ if (slot.params.cache_prompt) {+ for (size_t i = n_keep + n_discard; i < slot.cache_tokens.size(); i++) {+ slot.cache_tokens[i - n_discard] = slot.cache_tokens[i];+ }- slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard);+ slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard);+ }slot.n_past -= n_discard;@@ -1651,33 +1721,12 @@ struct llama_server_context}}- // decode any currently ongoing sequences- LOG_VERBOSE("decoding ongoing sequences", {});- for (auto & slot : slots)- {- // release the slot- if (slot.command == RELEASE)- {- slot.state = IDLE;- slot.command = NONE;- slot.t_last_used = ggml_time_us();-- LOG_INFO("slot released", {- {"slot_id", slot.id},- {"task_id", slot.task_id},- {"n_ctx", n_ctx},- {"n_past", slot.n_past},- {"n_system_tokens", system_tokens.size()},- {"n_cache_tokens", slot.cache_tokens.size()},- {"truncated", slot.truncated}- });- queue_tasks.notify_slot_changed();-- continue;- }+ // start populating the batch for this iteration+ llama_batch_clear(batch);- if (slot.state == IDLE)- {+ // frist, add sampled tokens from any ongoing sequences+ for (auto & slot : slots) {+ if (slot.state == SLOT_STATE_IDLE) {continue;}@@ -1687,187 +1736,217 @@ struct llama_server_context// TODO: we always have to take into account the "system_tokens"// this is not great and needs to be improved somehow- llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id }, true);+ llama_batch_add(batch, slot.sampled, system_tokens.size() + slot_npast, { slot.id + 1 }, true);+slot.n_past += 1;++ if (slot.params.cache_prompt) {+ slot.cache_tokens.push_back(slot.sampled);+ }++ LOG_VERBOSE("slot decode token", {+ {"id_slot", slot.id},+ {"id_task", slot.id_task},+ {"n_ctx", n_ctx},+ {"n_past", slot.n_past},+ {"n_system_tokens", system_tokens.size()},+ {"n_cache_tokens", slot.cache_tokens.size()},+ {"truncated", slot.truncated}+ });}// process in chunks of params.n_batch- int32_t n_batch = params.n_batch;+ int32_t n_batch = llama_n_batch(ctx);+ int32_t n_ubatch = llama_n_ubatch(ctx);++ // next, batch any pending prompts without exceeding n_batch+ if (params.cont_batching || batch.n_tokens == 0) {+ for (auto & slot : slots) {+ // this slot still has a prompt to be processed+ if (slot.state == SLOT_STATE_IDLE && slot.command == SLOT_COMMAND_LOAD_PROMPT) {+ auto & prompt_tokens = slot.prompt_tokens;++ // we haven't tokenized the prompt yet - do it now:+ if (prompt_tokens.empty()) {+ LOG_VERBOSE("tokenizing prompt", {+ {"id_slot", slot.id},+ {"id_task", slot.id_task}+ });- // assign workload to the slots- if (params.cont_batching || batch.n_tokens == 0)- {- for (auto & slot : slots)- {- const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get<std::string>().empty()) || !slot.images.empty();+ slot.t_start_process_prompt = ggml_time_us();+ slot.t_start_generation = 0;- // empty prompt passed -> release the slot and send empty response- // note: infill mode allows empty prompt- if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt && !slot.infill)- {- slot.release();- slot.print_timings();- send_final_response(slot);- continue;- }+ if (slot.infill) {+ bool suff_rm_leading_spc = true;+ if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1) {+ params.input_suffix.erase(0, 1);+ suff_rm_leading_spc = false;+ }- // need process the prompt- if (slot.state == IDLE && slot.command == LOAD_PROMPT)- {- slot.state = PROCESSING;- slot.command = NONE;- std::vector<llama_token> prompt_tokens;- slot.t_start_process_prompt = ggml_time_us();- slot.t_start_genereration = 0;-- if (slot.infill)- {- bool suff_rm_leading_spc = true;- if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1)- {- params.input_suffix.erase(0, 1);- suff_rm_leading_spc = false;+ auto prefix_tokens = tokenize(slot.params.input_prefix, false);+ auto suffix_tokens = tokenize(slot.params.input_suffix, false);++ const int space_token = 29871; // TODO: this should not be hardcoded+ if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {+ suffix_tokens.erase(suffix_tokens.begin());+ }++ prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));+ prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS+ prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));+ prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());+ prefix_tokens.push_back(llama_token_middle(model));+ prompt_tokens = prefix_tokens;+ } else {+ prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt}- auto prefix_tokens = tokenize(slot.params.input_prefix, false);- auto suffix_tokens = tokenize(slot.params.input_suffix, false);- const int space_token = 29871; // TODO: this should not be hardcoded- if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {- suffix_tokens.erase(suffix_tokens.begin());+ slot.n_past = 0;+ slot.n_prompt_tokens = prompt_tokens.size();++ LOG_VERBOSE("prompt tokenized", {+ {"id_slot", slot.id},+ {"id_task", slot.id_task},+ {"n_ctx", slot.n_ctx},+ {"n_keep", slot.params.n_keep},+ {"n_prompt_tokens", slot.n_prompt_tokens},+ {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())},+ });++ // empty prompt passed -> release the slot and send empty response+ if (prompt_tokens.empty()) {+ LOG_INFO("empty prompt - releasing slot", {+ {"id_slot", slot.id},+ {"id_task", slot.id_task}+ });++ slot.state = SLOT_STATE_PROCESSING;+ slot.command = SLOT_COMMAND_NONE;+ slot.release();+ slot.print_timings();+ send_final_response(slot);+ continue;}- prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));- prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS- prefix_tokens.insert(prefix_tokens.end(), llama_token_suffix(model));- prefix_tokens.insert(prefix_tokens.end(), suffix_tokens.begin(), suffix_tokens.end());- prefix_tokens.push_back(llama_token_middle(model));- prompt_tokens = prefix_tokens;- }- else- {- prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token); // add BOS if there isn't system prompt- }+ if (slot.embedding) {+ // this prompt is too large to process - discard it+ if (slot.n_prompt_tokens > n_ubatch) {+ slot.state = SLOT_STATE_PROCESSING;+ slot.command = SLOT_COMMAND_NONE;+ slot.release();+ slot.print_timings();+ send_final_response(slot);+ continue;+ }+ } else {+ if (slot.params.n_keep < 0) {+ slot.params.n_keep = slot.n_prompt_tokens;+ }+ slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep);- slot.num_prompt_tokens = prompt_tokens.size();+ // if input prompt is too big, truncate it (if group attention self-extend is disabled)+ if (slot.ga_n == 1 && slot.n_prompt_tokens >= slot.n_ctx) {+ const int n_left = slot.n_ctx - slot.params.n_keep;- if (slot.params.n_keep < 0)- {- slot.params.n_keep = slot.num_prompt_tokens;- }- slot.params.n_keep = std::min(slot.n_ctx - 4, slot.params.n_keep);-- // if input prompt is too big, truncate it- if (slot.num_prompt_tokens >= slot.n_ctx)- {- const int n_left = slot.n_ctx - slot.params.n_keep;- const int n_block_size = n_left / 2;- const int erased_blocks = (slot.num_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size;-- std::vector<llama_token> new_tokens(prompt_tokens.begin(), prompt_tokens.begin() + slot.params.n_keep);- new_tokens.insert(new_tokens.end(), prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size, prompt_tokens.end());-- LOG_VERBOSE("input truncated", {- {"n_ctx", slot.n_ctx},- {"n_keep", slot.params.n_keep},- {"n_left", n_left},- {"new_tokens", tokens_to_str(ctx, new_tokens.cbegin(), new_tokens.cend())},- });- slot.truncated = true;- prompt_tokens = new_tokens;+ const int n_block_size = n_left / 2;+ const int erased_blocks = (slot.n_prompt_tokens - slot.params.n_keep - n_block_size) / n_block_size;- slot.num_prompt_tokens = prompt_tokens.size();- GGML_ASSERT(slot.num_prompt_tokens < slot.n_ctx);- }+ std::vector<llama_token> new_tokens(+ prompt_tokens.begin(),+ prompt_tokens.begin() + slot.params.n_keep);- if (!slot.params.cache_prompt)- {- llama_sampling_reset(slot.ctx_sampling);+ new_tokens.insert(+ new_tokens.end(),+ prompt_tokens.begin() + slot.params.n_keep + erased_blocks * n_block_size,+ prompt_tokens.end());- slot.n_past = 0;- slot.n_past_se = 0;- slot.ga_i = 0;- slot.num_prompt_tokens_processed = slot.num_prompt_tokens;- }- else- {- // push the prompt into the sampling context (do not apply grammar)- for (auto &token : prompt_tokens)- {- llama_sampling_accept(slot.ctx_sampling, ctx, token, false);- }+ prompt_tokens = std::move(new_tokens);- slot.n_past = common_part(slot.cache_tokens, prompt_tokens);+ slot.truncated = true;+ slot.n_prompt_tokens = prompt_tokens.size();- // the last token of the cache is not in the KV cache until the next call to llama_decode- // (it was sampled, pushed into the "cache_tokens", but not yet put in the context)- if (slot.n_past > 0 && slot.n_past == (int32_t) slot.cache_tokens.size())- {- slot.n_past -= 1;- }+ LOG_VERBOSE("input truncated", {+ {"id_slot", slot.id},+ {"id_task", slot.id_task},+ {"n_ctx", slot.n_ctx},+ {"n_keep", slot.params.n_keep},+ {"n_left", n_left},+ {"n_prompt_tokens", slot.n_prompt_tokens},+ {"prompt_tokens", tokens_to_str(ctx, prompt_tokens.cbegin(), prompt_tokens.cend())},+ });++ GGML_ASSERT(slot.n_prompt_tokens < slot.n_ctx);+ }++ llama_sampling_reset(slot.ctx_sampling);++ if (!slot.params.cache_prompt) {+ slot.n_past_se = 0;+ slot.ga_i = 0;+ } else {+ GGML_ASSERT(slot.ga_n == 1);- slot.num_prompt_tokens_processed = slot.num_prompt_tokens - slot.n_past;-- if (slot.ga_n != 1)- {- int ga_i = 0;- int32_t ga_n = slot.ga_n;- int32_t ga_w = slot.ga_w;- int32_t slot_npast = 0;- for (int k = 0; k < slot.n_past; ++k)- {- while (slot_npast >= ga_i + ga_w) {- const int bd = (ga_w/ga_n)*(ga_n - 1);- slot_npast -= bd;- ga_i += ga_w/ga_n;+ // reuse any previously computed tokens that are common with the new prompt+ slot.n_past = common_part(slot.cache_tokens, prompt_tokens);++ // push the prompt into the sampling context (do not apply grammar)+ for (int i = 0; i < slot.n_past; ++i) {+ llama_sampling_accept(slot.ctx_sampling, ctx, slot.cache_tokens[i], false);}- slot_npast++;}- slot.n_past_se = slot_npast;- slot.ga_i = ga_i;}- LOG_INFO("slot progression", {- { "slot_id", slot.id },- { "task_id", slot.task_id },- { "n_past", slot.n_past },- { "num_prompt_tokens_processed", slot.num_prompt_tokens_processed }- });- }+ if (slot.n_past == slot.n_prompt_tokens && slot.n_past > 0) {+ // we have to evaluate at least 1 token to generate logits.+ LOG_INFO("we have to evaluate at least 1 token to generate logits", {+ { "id_slot", slot.id },+ { "id_task", slot.id_task }+ });- slot.cache_tokens = prompt_tokens;+ slot.n_past--;+ if (slot.ga_i > 0) {+ slot.n_past_se--;+ }+ }- if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0)- {- // we have to evaluate at least 1 token to generate logits.- LOG_INFO("we have to evaluate at least 1 token to generate logits", {- { "slot_id", slot.id },- { "task_id", slot.task_id }- });- slot.n_past--;- if (slot.ga_i > 0)- {- slot.n_past_se--;+ slot.n_prompt_tokens_processed = 0;+ }++ if (slot.embedding) {+ // cannot fit the prompt in the current batch - will try next iter+ if (batch.n_tokens + slot.n_prompt_tokens > n_batch) {+ continue;}}+ // keep only the common partint p0 = (int) system_tokens.size() + slot.n_past;- LOG_INFO("kv cache rm [p0, end)", {- { "slot_id", slot.id },- { "task_id", slot.task_id },- { "p0", p0 }- });- llama_kv_cache_seq_rm(ctx, slot.id, p0, -1);+ if (!llama_kv_cache_seq_rm(ctx, slot.id + 1, p0, -1)) {+ // could not partially delete (likely using a non-Transformer model)+ llama_kv_cache_seq_rm(ctx, slot.id + 1, -1, -1);++ p0 = (int) system_tokens.size();+ if (p0 != 0) {+ // copy over the system prompt when there is one+ llama_kv_cache_seq_cp(ctx, 0, slot.id + 1, -1, -1);+ }- LOG_VERBOSE("prompt ingested", {- {"n_past", slot.n_past},- {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},- {"to_eval", tokens_to_str(ctx, slot.cache_tokens.cbegin() + slot.n_past, slot.cache_tokens.cend())},- });+ // there is no common part left (except for the system prompt)+ slot.n_past = 0;+ slot.n_past_se = 0;+ slot.ga_i = 0;+ // TODO: is the system prompt ever in the sampling context?+ llama_sampling_reset(slot.ctx_sampling);+ }- const bool has_images = process_images(slot);+ // remove the non-common part from the cache+ slot.cache_tokens.resize(slot.n_past);- // process the prefix of first image- std::vector<llama_token> prefix_tokens = has_images ? tokenize(slot.images[0].prefix_prompt, add_bos_token) : prompt_tokens;+ LOG_INFO("kv cache rm [p0, end)", {+ { "id_slot", slot.id },+ { "id_task", slot.id_task },+ { "p0", p0 }+ });int32_t slot_npast = slot.n_past_se > 0 ? slot.n_past_se : slot.n_past;@@ -1875,61 +1954,81 @@ struct llama_server_contextint32_t ga_n = slot.ga_n;int32_t ga_w = slot.ga_w;- for (; slot.n_past < (int) prefix_tokens.size(); ++slot.n_past)- {- if (slot.ga_n != 1)- {+ // add prompt tokens for processing in the current batch+ // TODO: the self-extend stuff here is a mess - simplify and/or abstract it somehow+ for (; slot.n_past < slot.n_prompt_tokens && batch.n_tokens < n_batch; ++slot.n_past) {+ if (slot.ga_n != 1) {while (slot_npast >= ga_i + ga_w) {const int bd = (ga_w/ga_n)*(ga_n - 1);slot_npast -= bd;ga_i += ga_w/ga_n;}}- llama_batch_add(batch, prefix_tokens[slot.n_past], system_tokens.size() + slot_npast, {slot.id }, false);++ llama_batch_add(batch, prompt_tokens[slot.n_past], system_tokens.size() + slot_npast, { slot.id + 1 }, false);++ if (slot.params.cache_prompt) {+ slot.cache_tokens.push_back(prompt_tokens[slot.n_past]);+ }++ slot.n_prompt_tokens_processed++;slot_npast++;}- if (has_images && !ingest_images(slot, n_batch))- {- LOG_ERROR("failed processing images", {- "slot_id", slot.id,- "task_id", slot.task_id,- });- // FIXME @phymbert: to be properly tested- // early returning without changing the slot state will block the slot for ever- // no one at the moment is checking the return value- return false;- }+ LOG_VERBOSE("prompt processing progress", {+ {"id_slot", slot.id},+ {"n_past", slot.n_past},+ {"n_ctx", n_ctx},+ {"n_tokens", batch.n_tokens},+ {"progress", (float) slot.n_prompt_tokens_processed / slot.n_prompt_tokens},+ });- // extract the logits only for the last token- if (batch.n_tokens > 0)- {+ // entire prompt has been processed - start decoding new tokens+ if (slot.n_past == slot.n_prompt_tokens) {+ slot.state = SLOT_STATE_PROCESSING;+ slot.command = SLOT_COMMAND_NONE;++ GGML_ASSERT(batch.n_tokens > 0);++ // extract the logits only for the last tokenbatch.logits[batch.n_tokens - 1] = true;++ slot.n_decoded = 0;+ slot.i_batch = batch.n_tokens - 1;++ LOG_VERBOSE("prompt done", {+ {"id_slot", slot.id},+ {"n_past", slot.n_past},+ {"n_ctx", n_ctx},+ {"n_tokens", batch.n_tokens},+ });}+ }- slot.n_decoded = 0;- slot.i_batch = batch.n_tokens - 1;+ if (batch.n_tokens >= n_batch) {+ break;}}}- if (batch.n_tokens == 0)- {- all_slots_are_idle = true;- return true;+ if (batch.n_tokens == 0) {+ LOG_VERBOSE("no tokens to decode", {});+ return;}- for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch)- {- const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));+ LOG_VERBOSE("decoding batch", {+ {"n_tokens", batch.n_tokens},+ });- for (auto & slot : slots)- {- if (slot.ga_n != 1)- {+ // process the created batch of tokens+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) {+ const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);++ for (auto & slot : slots) {+ if (slot.ga_n != 1) {// context extension via Self-Extend- while (slot.n_past_se >= slot.ga_i + slot.ga_w)- {+ // TODO: simplify and/or abstract this+ while (slot.n_past_se >= slot.ga_i + slot.ga_w) {const int ib = (slot.ga_n * slot.ga_i) / slot.ga_w;const int bd = (slot.ga_w / slot.ga_n) * (slot.ga_n - 1);const int dd = (slot.ga_w / slot.ga_n) - ib * bd - slot.ga_w;@@ -1939,9 +2038,9 @@ struct llama_server_contextLOG_TEE("div: [%6d, %6d] / %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n, (slot.ga_i + ib * bd) / slot.ga_n, (slot.ga_i + ib * bd + slot.ga_w) / slot.ga_n);LOG_TEE("shift: [%6d, %6d] + %6d -> [%6d, %6d]\n", slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd, slot.ga_i + ib * bd + slot.ga_w + dd, slot.n_past_se + ib * bd + dd);- llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i, slot.n_past_se, ib * bd);- llama_kv_cache_seq_div(ctx, slot.id, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w,slot.ga_n);- llama_kv_cache_seq_add(ctx, slot.id, slot.ga_i + ib * bd + slot.ga_w,slot.n_past_se + ib * bd, dd);+ llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i, slot.n_past_se, ib * bd);+ llama_kv_cache_seq_div(ctx, slot.id + 1, slot.ga_i + ib * bd, slot.ga_i + ib * bd + slot.ga_w, slot.ga_n);+ llama_kv_cache_seq_add(ctx, slot.id + 1, slot.ga_i + ib * bd + slot.ga_w, slot.n_past_se + ib * bd, dd);slot.n_past_se -= bd;@@ -1949,12 +2048,12 @@ struct llama_server_contextLOG_TEE("\nn_past_old = %d, n_past = %d, ga_i = %d\n\n", slot.n_past_se + bd, slot.n_past_se, slot.ga_i);}+slot.n_past_se += n_tokens;}}- llama_batch batch_view =- {+ llama_batch batch_view = {n_tokens,batch.token + i,nullptr,@@ -1967,13 +2066,17 @@ struct llama_server_contextconst int ret = llama_decode(ctx, batch_view);- if (ret != 0)- {- if (n_batch == 1 || ret < 0)- {+ if (ret != 0) {+ if (n_batch == 1 || ret < 0) {// if you get here, it means the KV cache is full - try increasing it via the context sizeLOG_TEE("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret);- return false;+ for (auto & slot : slots) {+ slot.state = SLOT_STATE_PROCESSING;+ slot.command = SLOT_COMMAND_NONE;+ slot.release();+ send_error(slot, "Input prompt is too big compared to KV size. Please try increasing KV size.");+ }+ break; // break loop of n_batch}LOG_TEE("%s : failed to find free space in the KV cache, retrying with smaller n_batch = %d\n", __func__, n_batch / 2);@@ -1981,23 +2084,21 @@ struct llama_server_context// retry with half the batch size to try to find a free slot in the KV cachen_batch /= 2;i -= n_batch;- continue;++ continue; // continue loop of n_batch}- for (auto & slot : slots)- {- if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens))- {- continue;+ for (auto & slot : slots) {+ if (slot.state != SLOT_STATE_PROCESSING || slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) {+ continue; // continue loop of slots}// prompt evaluated for embedding- if (slot.embedding)- {- send_embedding(slot);+ if (slot.embedding) {+ send_embedding(slot, batch_view);slot.release();slot.i_batch = -1;- continue;+ continue; // continue loop of slots}completion_token_output result;@@ -2006,10 +2107,9 @@ struct llama_server_contextllama_sampling_accept(slot.ctx_sampling, ctx, id, true);slot.n_decoded += 1;- if (slot.n_decoded == 1)- {- slot.t_start_genereration = ggml_time_us();- slot.t_prompt_processing = (slot.t_start_genereration - slot.t_start_process_prompt) / 1e3;+ if (slot.n_decoded == 1) {+ slot.t_start_generation = ggml_time_us();+ slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3;metrics.on_prompt_eval(slot);}@@ -2017,19 +2117,19 @@ struct llama_server_contextresult.tok = id;const int32_t n_probs = slot.sparams.n_probs;- if (slot.sparams.temp <= 0 && n_probs > 0)- {+ if (slot.sparams.temp <= 0 && n_probs > 0) {// for llama_sample_token_greedy we need to sort candidatesllama_sample_softmax(ctx, &cur_p);}- for (size_t i = 0; i < std::min(cur_p.size, (size_t)n_probs); ++i)- {- result.probs.push_back({cur_p.data[i].id, cur_p.data[i].p});+ for (size_t i = 0; i < std::min(cur_p.size, (size_t) n_probs); ++i) {+ result.probs.push_back({+ cur_p.data[i].id,+ cur_p.data[i].p+ });}- if (!process_token(result, slot))- {+ if (!process_token(result, slot)) {slot.release();slot.print_timings();send_final_response(slot);@@ -2040,18 +2140,22 @@ struct llama_server_context}}- LOG_VERBOSE("slots updated", {});- return true;+ LOG_VERBOSE("run slots completed", {});}- void run_on_all_tasks_finished() {- update_slots();+ json model_meta() const {+ return json {+ {"vocab_type", llama_vocab_type (model)},+ {"n_vocab", llama_n_vocab (model)},+ {"n_ctx_train", llama_n_ctx_train (model)},+ {"n_embd", llama_n_embd (model)},+ {"n_params", llama_model_n_params(model)},+ {"size", llama_model_size (model)},+ };}};-static void server_print_usage(const char *argv0, const gpt_params ¶ms,- const server_params &sparams)-{+static void server_print_usage(const char * argv0, const gpt_params & params, const server_params & sparams) {printf("usage: %s [options]\n", argv0);printf("\n");printf("options:\n");@@ -2059,6 +2163,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled");printf(" -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);printf(" -tb N, --threads-batch N number of threads to use during batch and prompt processing (default: same as --threads)\n");+ printf(" --threads-http N number of threads in the http server pool to process requests (default: max(hardware concurrency - 1, --parallel N + 2))\n");printf(" -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx);printf(" --rope-scaling {none,linear,yarn}\n");printf(" RoPE frequency scaling method, defaults to linear unless specified by the model\n");@@ -2068,15 +2173,17 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n");printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow);printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast);- printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);+ printf(" --pooling {none,mean,cls} pooling type for embeddings, use model default if unspecified\n");+ printf(" -dt N, --defrag-thold N\n");+ printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold);+ printf(" -b N, --batch-size N logical maximum batch size (default: %d)\n", params.n_batch);+ printf(" -ub N, --ubatch-size N physical maximum batch size (default: %d)\n", params.n_ubatch);printf(" --memory-f32 use f32 instead of f16 for memory key+value (default: disabled)\n");printf(" not recommended: doubles context memory required and no measurable increase in quality\n");- if (llama_supports_mlock())- {+ if (llama_supports_mlock()) {printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");}- if (llama_supports_mmap())- {+ if (llama_supports_mmap()) {printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");}printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n");@@ -2098,17 +2205,23 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,}printf(" -m FNAME, --model FNAME\n");printf(" model path (default: %s)\n", params.model.c_str());+ printf(" -mu MODEL_URL, --model-url MODEL_URL\n");+ printf(" model download url (default: %s)\n", params.model_url.c_str());printf(" -a ALIAS, --alias ALIAS\n");printf(" set an alias for the model, will be added as `model` field in completion response\n");printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);- printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());+ printf(" --path PUBLIC_PATH path from which to serve static files (default: disabled)\n");printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT+ printf(" --ssl-key-file FNAME path to file a PEM-encoded SSL private key\n");+ printf(" --ssl-cert-file FNAME path to file a PEM-encoded SSL certificate\n");+#endifprintf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);- printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");+ printf(" --embeddings enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");printf(" -spf FNAME, --system-prompt-file FNAME\n");@@ -2117,7 +2230,6 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" KV cache data type for K (default: f16)\n");printf(" -ctv TYPE, --cache-type-v TYPE\n");printf(" KV cache data type for V (default: f16)\n");- printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");printf(" --log-format log output format: json or text (default: json)\n");printf(" --log-disable disables logging to a file.\n");printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n");@@ -2127,65 +2239,50 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" --override-kv KEY=TYPE:VALUE\n");printf(" advanced option to override model metadata by key. may be specified multiple times.\n");printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");- printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`");- printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`");+ printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`\n");+ printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`\n");printf(" --chat-template JINJA_TEMPLATE\n");printf(" set custom jinja chat template (default: template taken from model's metadata)\n");- printf(" Note: only commonly used templates are accepted, since we don't have jinja parser\n");+ printf(" only commonly used templates are accepted:\n");+ printf(" https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template\n");printf("\n");}-static void server_params_parse(int argc, char **argv, server_params &sparams,- gpt_params ¶ms, llama_server_context& llama)-{- gpt_params default_params;+static void server_params_parse(int argc, char ** argv, server_params & sparams, gpt_params & params) {+ gpt_params default_params;server_params default_sparams;+std::string arg;bool invalid_param = false;- for (int i = 1; i < argc; i++)- {+ for (int i = 1; i < argc; i++) {arg = argv[i];- if (arg == "--port")- {- if (++i >= argc)- {+ if (arg == "--port") {+ if (++i >= argc) {invalid_param = true;break;}sparams.port = std::stoi(argv[i]);- }- else if (arg == "--host")- {- if (++i >= argc)- {+ } else if (arg == "--host") {+ if (++i >= argc) {invalid_param = true;break;}sparams.hostname = argv[i];- }- else if (arg == "--path")- {- if (++i >= argc)- {+ } else if (arg == "--path") {+ if (++i >= argc) {invalid_param = true;break;}sparams.public_path = argv[i];- }- else if (arg == "--api-key")- {- if (++i >= argc)- {+ } else if (arg == "--api-key") {+ if (++i >= argc) {invalid_param = true;break;}- sparams.api_keys.emplace_back(argv[i]);- }- else if (arg == "--api-key-file")- {- if (++i >= argc)- {+ sparams.api_keys.push_back(argv[i]);+ } else if (arg == "--api-key-file") {+ if (++i >= argc) {invalid_param = true;break;}@@ -2202,53 +2299,59 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}}key_file.close();+}- else if (arg == "--timeout" || arg == "-to")- {- if (++i >= argc)- {+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT+ else if (arg == "--ssl-key-file") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ sparams.ssl_key_file = argv[i];+ } else if (arg == "--ssl-cert-file") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ sparams.ssl_cert_file = argv[i];+ }+#endif+ else if (arg == "--timeout" || arg == "-to") {+ if (++i >= argc) {invalid_param = true;break;}sparams.read_timeout = std::stoi(argv[i]);sparams.write_timeout = std::stoi(argv[i]);- }- else if (arg == "-m" || arg == "--model")- {- if (++i >= argc)- {+ } else if (arg == "-m" || arg == "--model") {+ if (++i >= argc) {invalid_param = true;break;}params.model = argv[i];- }- else if (arg == "-a" || arg == "--alias")- {- if (++i >= argc)- {+ } else if (arg == "-mu" || arg == "--model-url") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.model_url = argv[i];+ } else if (arg == "-a" || arg == "--alias") {+ if (++i >= argc) {invalid_param = true;break;}params.model_alias = argv[i];- }- else if (arg == "-h" || arg == "--help")- {+ } else if (arg == "-h" || arg == "--help") {server_print_usage(argv[0], default_params, default_sparams);exit(0);- }- else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size")- {- if (++i >= argc)- {+ } else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size") {+ if (++i >= argc) {invalid_param = true;break;}params.n_ctx = std::stoi(argv[i]);- }- else if (arg == "--rope-scaling")- {- if (++i >= argc)- {+ } else if (arg == "--rope-scaling") {+ if (++i >= argc) {invalid_param = true;break;}@@ -2257,150 +2360,142 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }else { invalid_param = true; break; }- }- else if (arg == "--rope-freq-base")- {- if (++i >= argc)- {+ } else if (arg == "--rope-freq-base") {+ if (++i >= argc) {invalid_param = true;break;}params.rope_freq_base = std::stof(argv[i]);- }- else if (arg == "--rope-freq-scale")- {- if (++i >= argc)- {+ } else if (arg == "--rope-freq-scale") {+ if (++i >= argc) {invalid_param = true;break;}params.rope_freq_scale = std::stof(argv[i]);- }- else if (arg == "--yarn-ext-factor")- {+ } else if (arg == "--yarn-ext-factor") {if (++i >= argc) {invalid_param = true;break;}params.yarn_ext_factor = std::stof(argv[i]);}- else if (arg == "--yarn-attn-factor")- {+ else if (arg == "--yarn-attn-factor") {if (++i >= argc) {invalid_param = true;break;}params.yarn_attn_factor = std::stof(argv[i]);- }- else if (arg == "--yarn-beta-fast")- {+ } else if (arg == "--yarn-beta-fast") {if (++i >= argc) {invalid_param = true;break;}params.yarn_beta_fast = std::stof(argv[i]);- }- else if (arg == "--yarn-beta-slow")- {+ } else if (arg == "--yarn-beta-slow") {if (++i >= argc) {invalid_param = true;break;}params.yarn_beta_slow = std::stof(argv[i]);- }- else if (arg == "--threads" || arg == "-t")- {+ } else if (arg == "--pooling") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ std::string value(argv[i]);+ /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }+ else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }+ else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }+ else { invalid_param = true; break; }+ } else if (arg == "--defrag-thold" || arg == "-dt") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.defrag_thold = std::stof(argv[i]);+ } else if (arg == "--threads" || arg == "-t") {if (++i >= argc){invalid_param = true;break;}params.n_threads = std::stoi(argv[i]);- }- else if (arg == "--grp-attn-n" || arg == "-gan")- {+ } else if (arg == "--grp-attn-n" || arg == "-gan") {if (++i >= argc) {invalid_param = true;break;}params.grp_attn_n = std::stoi(argv[i]);- }- else if (arg == "--grp-attn-w" || arg == "-gaw")- {- if (++i >= argc)- {+ } else if (arg == "--grp-attn-w" || arg == "-gaw") {+ if (++i >= argc) {invalid_param = true;break;}params.grp_attn_w = std::stoi(argv[i]);- }- else if (arg == "--threads-batch" || arg == "-tb")- {- if (++i >= argc)- {+ } else if (arg == "--threads-batch" || arg == "-tb") {+ if (++i >= argc) {invalid_param = true;break;}params.n_threads_batch = std::stoi(argv[i]);- }- else if (arg == "-b" || arg == "--batch-size")- {- if (++i >= argc)- {+ } else if (arg == "--threads-http") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ sparams.n_threads_http = std::stoi(argv[i]);+ } else if (arg == "-b" || arg == "--batch-size") {+ if (++i >= argc) {invalid_param = true;break;}params.n_batch = std::stoi(argv[i]);- params.n_batch = std::min(512, params.n_batch);- }- else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers")- {- if (++i >= argc)- {+ } else if (arg == "-ub" || arg == "--ubatch-size") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.n_ubatch = std::stoi(argv[i]);+ } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {+ if (++i >= argc) {invalid_param = true;break;}if (llama_supports_gpu_offload()) {params.n_gpu_layers = std::stoi(argv[i]);} else {- LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "- "See main README.md for information on enabling GPU BLAS support",- {{"n_gpu_layers", params.n_gpu_layers}});+ LOG_WARNING(+ "Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "+ "See main README.md for information on enabling GPU BLAS support",+ {{"n_gpu_layers", params.n_gpu_layers}});}- }- else if (arg == "--split-mode" || arg == "-sm")- {+ } else if (arg == "--split-mode" || arg == "-sm") {if (++i >= argc) {invalid_param = true;break;}std::string arg_next = argv[i];- if (arg_next == "none")- {+ if (arg_next == "none") {params.split_mode = LLAMA_SPLIT_MODE_NONE;- }- else if (arg_next == "layer")- {+ } else if (arg_next == "layer") {params.split_mode = LLAMA_SPLIT_MODE_LAYER;- }- else if (arg_next == "row")- {+ } else if (arg_next == "row") {params.split_mode = LLAMA_SPLIT_MODE_ROW;- }- else {+ } else {invalid_param = true;break;}- }- else if (arg == "--tensor-split" || arg == "-ts")- {- if (++i >= argc)- {+#ifndef GGML_USE_CUBLAS+ fprintf(stderr, "warning: llama.cpp was compiled without cuBLAS. Setting the split mode has no effect.\n");+#endif // GGML_USE_CUBLAS+ } else if (arg == "--tensor-split" || arg == "-ts") {+ if (++i >= argc) {invalid_param = true;break;}+#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL)std::string arg_next = argv[i];// split string by , and /@@ -2409,95 +2504,67 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,std::vector<std::string> split_arg{it, {}};GGML_ASSERT(split_arg.size() <= llama_max_devices());- for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device)- {- if (i_device < split_arg.size())- {+ for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device) {+ if (i_device < split_arg.size()) {params.tensor_split[i_device] = std::stof(split_arg[i_device]);- }- else- {+ } else {params.tensor_split[i_device] = 0.0f;}}- }- else if (arg == "--no-mul-mat-q" || arg == "-nommq")- {- params.mul_mat_q = false;- }- else if (arg == "--main-gpu" || arg == "-mg")- {- if (++i >= argc)- {+#else+ LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a tensor split.\n", {});+#endif // GGML_USE_CUBLAS+ } else if (arg == "--main-gpu" || arg == "-mg") {+ if (++i >= argc) {invalid_param = true;break;}+#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_SYCL)params.main_gpu = std::stoi(argv[i]);- }- else if (arg == "--lora")- {- if (++i >= argc)- {+#else+ LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a main GPU.", {});+#endif+ } else if (arg == "--lora") {+ if (++i >= argc) {invalid_param = true;break;}params.lora_adapter.emplace_back(argv[i], 1.0f);params.use_mmap = false;- }- else if (arg == "--lora-scaled")- {- if (++i >= argc)- {+ } else if (arg == "--lora-scaled") {+ if (++i >= argc) {invalid_param = true;break;}const char * lora_adapter = argv[i];- if (++i >= argc)- {+ if (++i >= argc) {invalid_param = true;break;}params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));params.use_mmap = false;- }- else if (arg == "--lora-base")- {- if (++i >= argc)- {+ } else if (arg == "--lora-base") {+ if (++i >= argc) {invalid_param = true;break;}params.lora_base = argv[i];- }- else if (arg == "-v" || arg == "--verbose")- {+ } else if (arg == "-v" || arg == "--verbose") {#if SERVER_VERBOSE != 1LOG_WARNING("server.cpp is not built with verbose logging.", {});#elseserver_verbose = true;#endif- }- else if (arg == "--nobrowser")- {+ } else if (arg == "--nobrowser") {sparams.nobrowser = true;- }- else if (arg == "--unsecure")- {+ } else if (arg == "--unsecure") {FLAG_unsecure = true;- }- else if (arg == "--server")- {- }- else if (arg == "--nocompile")- {+ } else if (arg == "--server") {+ } else if (arg == "--nocompile") {FLAG_nocompile = true;- }- else if (arg == "--recompile")- {+ } else if (arg == "--recompile") {FLAG_recompile = true;- }- else if (arg == "--gpu")- {+ } else if (arg == "--gpu") {if (++i >= argc){invalid_param = true;@@ -2509,55 +2576,39 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);exit(1);}- }- else if (arg == "--mlock")- {+ } else if (arg == "--mlock") {params.use_mlock = true;- }- else if (arg == "--no-mmap")- {+ } else if (arg == "--no-mmap") {params.use_mmap = false;- }- else if (arg == "--numa") {+ } else if (arg == "--numa") {if (++i >= argc) {invalid_param = true;break;} else {std::string value(argv[i]);/**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }- else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }- else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }+ else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }+ else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }else { invalid_param = true; break; }}- }- else if (arg == "--embedding")- {+ } else if (arg == "--embedding" || arg == "--embeddings") {params.embedding = true;- }- else if (arg == "-cb" || arg == "--cont-batching")- {+ } else if (arg == "-cb" || arg == "--cont-batching") {params.cont_batching = true;- }- else if (arg == "-np" || arg == "--parallel")- {- if (++i >= argc)- {+ } else if (arg == "-np" || arg == "--parallel") {+ if (++i >= argc) {invalid_param = true;break;}params.n_parallel = std::stoi(argv[i]);- } else if (arg == "-n" || arg == "--n-predict")- {- if (++i >= argc)- {+ } else if (arg == "-n" || arg == "--n-predict") {+ if (++i >= argc) {invalid_param = true;break;}params.n_predict = std::stoi(argv[i]);- } else if (arg == "-spf" || arg == "--system-prompt-file")- {- if (++i >= argc)- {+ } else if (arg == "-spf" || arg == "--system-prompt-file") {+ if (++i >= argc) {invalid_param = true;break;}@@ -2567,67 +2618,39 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}- std::string systm_content;+ std::string system_prompt;std::copy(std::istreambuf_iterator<char>(file),std::istreambuf_iterator<char>(),- std::back_inserter(systm_content)+ std::back_inserter(system_prompt));- llama.process_system_prompt_data(json::parse(systm_content));- }- else if (arg == "-ctk" || arg == "--cache-type-k") {+ sparams.system_prompt = system_prompt;+ } else if (arg == "-ctk" || arg == "--cache-type-k") {params.cache_type_k = argv[++i];- }- else if (arg == "-ctv" || arg == "--cache-type-v") {+ } else if (arg == "-ctv" || arg == "--cache-type-v") {params.cache_type_v = argv[++i];- }- else if(arg == "--mmproj")- {- if (++i >= argc)- {- invalid_param = true;- break;- }- params.mmproj = argv[i];- }- else if (arg == "--log-format")- {- if (++i >= argc)- {+ } else if (arg == "--log-format") {+ if (++i >= argc) {invalid_param = true;break;}- if (std::strcmp(argv[i], "json") == 0)- {+ if (std::strcmp(argv[i], "json") == 0) {server_log_json = true;- }- else if (std::strcmp(argv[i], "text") == 0)- {+ } else if (std::strcmp(argv[i], "text") == 0) {server_log_json = false;- }- else- {+ } else {invalid_param = true;break;}- }- else if (arg == "--log-disable")- {+ } else if (arg == "--log-disable") {log_set_target(stdout);LOG_INFO("logging to file is disabled.", {});- }- else if (arg == "--slots-endpoint-disable")- {+ } else if (arg == "--slots-endpoint-disable") {sparams.slots_endpoint = false;- }- else if (arg == "--metrics")- {+ } else if (arg == "--metrics") {sparams.metrics_endpoint = true;- }- else if (arg == "--chat-template")- {- if (++i >= argc)- {+ } else if (arg == "--chat-template") {+ if (++i >= argc) {invalid_param = true;break;}@@ -2638,9 +2661,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,break;}sparams.chat_template = argv[i];- }- else if (arg == "--override-kv")- {+ } else if (arg == "--override-kv") {if (++i >= argc) {invalid_param = true;break;@@ -2651,6 +2672,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}+struct llama_model_kv_override kvo;std::strncpy(kvo.key, argv[i], sep - argv[i]);kvo.key[sep - argv[i]] = 0;@@ -2681,71 +2703,28 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,break;}params.kv_overrides.push_back(kvo);- }- else- {+ } else {fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());server_print_usage(argv[0], default_params, default_sparams);exit(1);}}- params.embedding = true; // [jart] #243 always enable embedding mode- params.n_gpu_layers = llamafile_gpu_layers(params.n_gpu_layers);-if (!params.kv_overrides.empty()) {params.kv_overrides.emplace_back();params.kv_overrides.back().key[0] = 0;}- if (invalid_param)- {+ if (invalid_param) {fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());server_print_usage(argv[0], default_params, default_sparams);exit(1);}}-/* llama.cpp completion api semantics */-static json format_partial_response(- llama_server_context &llama, llama_client_slot *slot, const std::string &content, const std::vector<completion_token_output> &probs-) {- json res = json- {- {"content", content },- {"stop", false},- {"slot_id", slot->id },- {"multimodal", llama.multimodal }- };-- if (slot->sparams.n_probs > 0)- {- res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);- }-- return res;-}--static json format_tokenizer_response(const std::vector<llama_token> &tokens)-{- return json {- {"tokens", tokens}- };-}--static json format_detokenized_response(std::string content)-{- return json {- {"content", content}- };-}---static void log_server_request(const httplib::Request &req, const httplib::Response &res)-{+static void log_server_request(const httplib::Request & req, const httplib::Response & res) {// skip GH copilot requests when using default port- if (req.path == "/v1/health" || req.path == "/v1/completions")- {+ if (req.path == "/v1/health" || req.path == "/v1/completions") {return;}@@ -2764,296 +2743,155 @@ static void log_server_request(const httplib::Request &req, const httplib::Respo});}-struct token_translator-{- llama_context * ctx;- std::string operator()(llama_token tok) const { return llama_token_to_piece(ctx, tok); }- std::string operator()(const completion_token_output &cto) const { return (*this)(cto.tok); }-};+std::function<void(int)> shutdown_handler;+std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;-static void append_to_generated_text_from_generated_token_probs(llama_server_context &llama, llama_client_slot *slot)-{- auto & gtps = slot->generated_token_probs;- auto translator = token_translator{llama.ctx};- auto add_strlen = [=](size_t sum, const completion_token_output & cto) { return sum + translator(cto).size(); };- const size_t len = std::accumulate(gtps.begin(), gtps.end(), size_t(0), add_strlen);- if (slot->generated_text.capacity() < slot->generated_text.size() + len)- {- slot->generated_text.reserve(slot->generated_text.size() + len);- }- for (const completion_token_output & cto : gtps)- {- slot->generated_text += translator(cto);+inline void signal_handler(int signal) {+ if (is_terminating.test_and_set()) {+ // in case it hangs, we can force terminate the server by hitting Ctrl+C twice+ // this is for better developer experience, we can remove when the server is stable enough+ fprintf(stderr, "Received second interrupt, terminating immediately.\n");+ exit(1);}-}-std::function<void(int)> shutdown_handler;-inline void signal_handler(int signal) { shutdown_handler(signal); }+ shutdown_handler(signal);+}static const char *sockaddr2str(const struct sockaddr *sa, char *buf, size_t size) {- if (sa->sa_family == AF_INET) {- return inet_ntop(AF_INET, &(((const struct sockaddr_in *)sa)->sin_addr),- buf, size);- } else if (sa->sa_family == AF_INET6) {- return inet_ntop(AF_INET6, &(((const struct sockaddr_in6 *)sa)->sin6_addr),- buf, size);- } else {- return 0;- }+ if (sa->sa_family == AF_INET) {+ return inet_ntop(AF_INET, &(((const struct sockaddr_in *)sa)->sin_addr),+ buf, size);+ } else if (sa->sa_family == AF_INET6) {+ return inet_ntop(AF_INET6, &(((const struct sockaddr_in6 *)sa)->sin6_addr),+ buf, size);+ } else {+ return 0;+ }}-int server_cli(int argc, char **argv)-{+int server_cli(int argc, char ** argv) {#if SERVER_VERBOSE != 1log_disable();#endif// own arguments required by this example- gpt_params params;+ gpt_params params;server_params sparams;// struct that contains llama context and inference- llama_server_context llama;+ server_context ctx_server;- server_params_parse(argc, argv, sparams, params, llama);+ server_params_parse(argc, argv, sparams, params);- if (params.model_alias == "unknown")- {+ if (!sparams.system_prompt.empty()) {+ ctx_server.system_prompt_set(json::parse(sparams.system_prompt));+ }++ if (params.model_alias == "unknown") {params.model_alias = params.model;}llama_backend_init();llama_numa_init(params.numa);- LOG_INFO("build info", {{"build", LLAMA_BUILD_NUMBER},- {"commit", LLAMA_COMMIT}});+ LOG_INFO("build info", {+ {"build", LLAMA_BUILD_NUMBER},+ {"commit", LLAMA_COMMIT}+ });LOG_INFO("system info", {- {"n_threads", params.n_threads},- {"n_threads_batch", params.n_threads_batch},- {"total_threads", std::thread::hardware_concurrency()},- {"system_info", llama_print_system_info()},- });+ {"n_threads", params.n_threads},+ {"n_threads_batch", params.n_threads_batch},+ {"total_threads", std::thread::hardware_concurrency()},+ {"system_info", llama_print_system_info()},+ });- httplib::Server svr;+ std::unique_ptr<httplib::Server> svr;+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT+ if (sparams.ssl_key_file != "" && sparams.ssl_cert_file != "") {+ LOG_INFO("Running with SSL", {{"key", sparams.ssl_key_file}, {"cert", sparams.ssl_cert_file}});+ svr.reset(+ new httplib::SSLServer(sparams.ssl_cert_file.c_str(), sparams.ssl_key_file.c_str())+ );+ } else {+ LOG_INFO("Running without SSL", {});+ svr.reset(new httplib::Server());+ }+#else+ svr.reset(new httplib::Server());+#endifstd::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};- svr.set_default_headers({{"Server", "llama.cpp"}});+ svr->set_default_headers({{"Server", "llama.cpp"}});// CORS preflight- svr.Options(R"(.*)", [](const httplib::Request &req, httplib::Response &res) {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ svr->Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));res.set_header("Access-Control-Allow-Credentials", "true");- res.set_header("Access-Control-Allow-Methods", "POST");- res.set_header("Access-Control-Allow-Headers", "*");+ res.set_header("Access-Control-Allow-Methods", "POST");+ res.set_header("Access-Control-Allow-Headers", "*");+ return res.set_content("", "application/json; charset=utf-8");});- svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) {- server_state current_state = state.load();- switch(current_state) {- case SERVER_STATE_READY: {- // request slots data using task queue- task_server task;- task.id = llama.queue_tasks.get_new_id();- task.type = TASK_TYPE_METRICS;- task.target_id = -1;-- llama.queue_results.add_waiting_task_id(task.id);- llama.queue_tasks.post(task);+ svr->set_logger(log_server_request);- // get the result- task_result result = llama.queue_results.recv(task.id);- llama.queue_results.remove_waiting_task_id(task.id);+ auto res_error = [](httplib::Response & res, json error_data) {+ json final_response {{"error", error_data}};+ res.set_content(final_response.dump(), "application/json; charset=utf-8");+ res.status = json_value(error_data, "code", 500);+ };- int n_idle_slots = result.result_json["idle"];- int n_processing_slots = result.result_json["processing"];+ svr->set_exception_handler([&res_error](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) {+ std::string message;+ try {+ std::rethrow_exception(std::move(ep));+ } catch (std::exception & e) {+ message = e.what();+ } catch (...) {+ message = "Unknown Exception";+ }- json health = {- {"status", "ok"},- {"slots_idle", n_idle_slots},- {"slots_processing", n_processing_slots}};- res.status = 200; // HTTP OK- if (sparams.slots_endpoint && req.has_param("include_slots")) {- health["slots"] = result.result_json["slots"];- }+ json formatted_error = format_error_response(message, ERROR_TYPE_SERVER);+ LOG_VERBOSE("Got exception", formatted_error);+ res_error(res, formatted_error);+ });- if (n_idle_slots == 0) {- health["status"] = "no slot available";- if (req.has_param("fail_on_no_slot")) {- res.status = 503; // HTTP Service Unavailable- }- }- res.set_content(health.dump(), "application/json");- break;- }- case SERVER_STATE_LOADING_MODEL:- res.set_content(R"({"status": "loading model"})", "application/json");- res.status = 503; // HTTP Service Unavailable- break;- case SERVER_STATE_ERROR:- res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json");- res.status = 500; // HTTP Internal Server Error- break;+ svr->set_error_handler([&res_error](const httplib::Request &, httplib::Response & res) {+ if (res.status == 404) {+ res_error(res, format_error_response("File Not Found", ERROR_TYPE_NOT_FOUND));}+ // for other error codes, we skip processing here because it's already done by res_error()});- if (sparams.slots_endpoint) {- svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) {- // request slots data using task queue- task_server task;- task.id = llama.queue_tasks.get_new_id();- task.type = TASK_TYPE_METRICS;- task.target_id = -1;+ // set timeouts and change hostname and port+ svr->set_read_timeout (sparams.read_timeout);+ svr->set_write_timeout(sparams.write_timeout);- llama.queue_results.add_waiting_task_id(task.id);- llama.queue_tasks.post(task);+ std::unordered_map<std::string, std::string> log_data;- // get the result- task_result result = llama.queue_results.recv(task.id);- llama.queue_results.remove_waiting_task_id(task.id);+ log_data["hostname"] = sparams.hostname;+ log_data["port"] = std::to_string(sparams.port);- res.set_content(result.result_json["slots"].dump(), "application/json");- res.status = 200; // HTTP OK- });+ if (sparams.api_keys.size() == 1) {+ auto key = sparams.api_keys[0];+ log_data["api_key"] = "api_key: ****" + key.substr(std::max((int)(key.length() - 4), 0));+ } else if (sparams.api_keys.size() > 1) {+ log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";}- if (sparams.metrics_endpoint) {- svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) {- // request slots data using task queue- task_server task;- task.id = llama.queue_tasks.get_new_id();- task.type = TASK_TYPE_METRICS;- task.target_id = -1;-- llama.queue_results.add_waiting_task_id(task.id);- llama.queue_tasks.post(task);+ // load the model+ if (!ctx_server.load_model(params)) {+ state.store(SERVER_STATE_ERROR);+ return 1;+ } else {+ ctx_server.init();+ state.store(SERVER_STATE_READY);+ }- // get the result- task_result result = llama.queue_results.recv(task.id);- llama.queue_results.remove_waiting_task_id(task.id);-- json data = result.result_json;-- uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"];- uint64_t t_prompt_processing = data["t_prompt_processing"];-- uint64_t n_tokens_predicted = data["n_tokens_predicted"];- uint64_t t_tokens_generation = data["t_tokens_generation"];-- int32_t kv_cache_used_cells = data["kv_cache_used_cells"];-- // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names- json all_metrics_def = json {- {"counter", {{- {"name", "prompt_tokens_total"},- {"help", "Number of prompt tokens processed."},- {"value", data["n_prompt_tokens_processed_total"]}- }, {- {"name", "tokens_predicted_total"},- {"help", "Number of generation tokens processed."},- {"value", data["n_tokens_predicted_total"]}- }}},- {"gauge", {{- {"name", "prompt_tokens_seconds"},- {"help", "Average prompt throughput in tokens/s."},- {"value", n_prompt_tokens_processed ? 1e3 / t_prompt_processing * n_prompt_tokens_processed : 0}- },{- {"name", "predicted_tokens_seconds"},- {"help", "Average generation throughput in tokens/s."},- {"value", n_tokens_predicted ? 1e3 / t_tokens_generation * n_tokens_predicted : 0}- },{- {"name", "kv_cache_usage_ratio"},- {"help", "KV-cache usage. 1 means 100 percent usage."},- {"value", 1. * kv_cache_used_cells / params.n_ctx}- },{- {"name", "kv_cache_tokens"},- {"help", "KV-cache tokens."},- {"value", data["kv_cache_tokens_count"]}- },{- {"name", "requests_processing"},- {"help", "Number of request processing."},- {"value", data["processing"]}- },{- {"name", "requests_deferred"},- {"help", "Number of request deferred."},- {"value", data["deferred"]}- }}}- };-- std::stringstream prometheus;- for (const auto& el : all_metrics_def.items()) {- const auto& type = el.key();- const auto& metrics_def = el.value();- for (const auto& metric_def : metrics_def) {- std::string name = metric_def["name"];- std::string help = metric_def["help"];- prometheus << "# HELP llamacpp:" << name << " " << help << "\n"- << "# TYPE llamacpp:" << name << " " << type << "\n"- << "llamacpp:" << name << " " << metric_def["value"] << "\n";- }- }-- res.set_content(prometheus.str(), "text/plain; version=0.0.4");- res.status = 200; // HTTP OK- });- }-- svr.set_logger(log_server_request);-- svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep)- {- const char fmt[] = "500 Internal Server Error\n%s";- char buf[BUFSIZ];- try- {- std::rethrow_exception(std::move(ep));- }- catch (std::exception &e)- {- snprintf(buf, sizeof(buf), fmt, e.what());- }- catch (...)- {- snprintf(buf, sizeof(buf), fmt, "Unknown Exception");- }- res.set_content(buf, "text/plain; charset=utf-8");- res.status = 500;- });-- svr.set_error_handler([](const httplib::Request &, httplib::Response &res)- {- if (res.status == 401)- {- res.set_content("Unauthorized", "text/plain; charset=utf-8");- }- if (res.status == 400)- {- res.set_content("Invalid request", "text/plain; charset=utf-8");- }- else if (res.status == 404)- {- res.set_content("File Not Found", "text/plain; charset=utf-8");- res.status = 404;- }- });-- // load the model- if (!llama.load_model(params))- {- state.store(SERVER_STATE_ERROR);- return 1;- } else {- llama.initialize();- state.store(SERVER_STATE_READY);- LOG_INFO("model loaded", {});- }-- // set timeouts and change hostname and port- svr.set_read_timeout (sparams.read_timeout);- svr.set_write_timeout(sparams.write_timeout);+ LOG_INFO("model loaded", {});for (int i = 0;; ++i) {- if (svr.bind_to_port(sparams.hostname, sparams.port)) {+ if (svr->bind_to_port(sparams.hostname, sparams.port)) {break;} else if (i < 10) {++sparams.port;@@ -3063,9 +2901,6 @@ int server_cli(int argc, char **argv)}}- // Set the base directory for serving static files- svr.set_base_dir(sparams.public_path);-// to make it ctrl+clickable:const char *connect_host;if (sparams.hostname != "0.0.0.0") {@@ -3094,107 +2929,66 @@ int server_cli(int argc, char **argv)}}- std::unordered_map<std::string, std::string> log_data;- log_data["hostname"] = sparams.hostname;- log_data["port"] = std::to_string(sparams.port);+ const auto model_meta = ctx_server.model_meta();- if (sparams.api_keys.size() == 1) {- log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4);- } else if (sparams.api_keys.size() > 1) {- log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";+ // if a custom chat template is not supplied, we will use the one that comes with the model (if any)+ if (sparams.chat_template.empty()) {+ if (!ctx_server.validate_model_chat_template()) {+ LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {});+ sparams.chat_template = "chatml";+ }}- LOG_INFO("HTTP server listening", log_data);- // run the HTTP server in a thread - see comment below- std::thread t([&]()- {- if (!svr.listen_after_bind())- {- state.store(SERVER_STATE_ERROR);- return 1;- }-- return 0;- });+ // print sample chat example to make it clear which template is used+ {+ json chat;+ chat.push_back({{"role", "system"}, {"content", "You are a helpful assistant"}});+ chat.push_back({{"role", "user"}, {"content", "Hello"}});+ chat.push_back({{"role", "assistant"}, {"content", "Hi there"}});+ chat.push_back({{"role", "user"}, {"content", "How are you?"}});- // launch browser tab- if (!sparams.nobrowser) {- char url[128];- snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);- llamafile_launch_browser(url);- }+ const std::string chat_example = format_chat(ctx_server.model, sparams.chat_template, chat);- if (sparams.chat_template.empty()) { // custom chat template is not supplied- // check if the template comes with the model is supported by us- llama.validate_model_chat_template(sparams);+ LOG_INFO("chat template", {+ {"chat_example", chat_example},+ {"built_in", sparams.chat_template.empty()},+ });}- if (!FLAG_unsecure) {- if (IsXnu()) {- // Cosmopolitan libc explicitly does not support cosmo_dlopen on x64- // macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt- // to enable sandboxing and ignore the failure on x64 macOS.- // (Alternatively, the sandbox-exec(1) command could be used to- // launch llamafile with a sandbox profile.) A goal of llamafile is- // to work easily out-of-the-box and it already operates without a- // sandbox on macOS so don't require --unsecure on x64 macOS to- // workaround the failure. If support is added later, it will be- // used without further changes. On Apple Silicon, if sandbox- // initialization fails, exit and recommend using --unsecure to- // disable sandboxing.- std::string error_msg;- int rv = mac_sandbox_init(error_msg);- if (rv != 0) {- fprintf(stderr, "Sandbox initialization failed:\n"- "rv: %d, error: %s.\n"- "Disable sandboxing with the --unsecure option.\n",- rv, error_msg.c_str());- if (IsXnuSilicon()) {- exit(1);- } else {- LOG_WARNING("Sandbox initialization failed. Not supported "- "on x64 macOS. Continuing.", {});- }- }- } else if (!llamafile_has_gpu()) {- printf("In the sandboxing block!\n");- // Enables pledge() security on Linux and OpenBSD.- // - We do this *after* binding the server socket.- // - We do this *after* opening the log file for writing.- // - We do this *after* opening a tab in the user browser.- // - We do this *before* loading any weights or graphdefs.- // In effect, what this does is:- // - Filesystem access is disabled entirely (except ZipOS).- // - On Linux, network access is restricted to accept() only.- // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.- char promises[32];- if (IsOpenbsd()) {- strlcpy(promises, "stdio inet", sizeof(promises));- } else {- strlcpy(promises, "stdio anet", sizeof(promises));- }- if (!startswith(sparams.public_path.c_str(), "/zip/")) {- strlcat(promises, " rpath", sizeof(promises));- }- __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;- if (pledge(0, 0)) {- LOG_TEE("warning: this OS doesn't support pledge() security\n");- } else if (pledge(promises, 0)) {- perror("pledge");- exit(1);- }- }- }+ //+ // Middlewares+ //++ auto middleware_validate_api_key = [&sparams, &res_error](const httplib::Request & req, httplib::Response & res) {+ // TODO: should we apply API key to all endpoints, including "/health" and "/models"?+ static const std::set<std::string> protected_endpoints = {+ "/props",+ "/completion",+ "/completions",+ "/v1/completions",+ "/chat/completions",+ "/v1/chat/completions",+ "/infill",+ "/tokenize",+ "/detokenize",+ "/embedding",+ "/embeddings",+ "/v1/embeddings",+ };- // Middleware for API key validation- auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool {// If API key is not set, skip validationif (sparams.api_keys.empty()) {return true;}+ // If path is not in protected_endpoints list, skip validation+ if (protected_endpoints.find(req.path) == protected_endpoints.end()) {+ return true;+ }+// Check for API key in the headerauto auth_header = req.get_header_value("Authorization");+std::string prefix = "Bearer ";if (auth_header.substr(0, prefix.size()) == prefix) {std::string received_api_key = auth_header.substr(prefix.size());@@ -3204,427 +2998,663 @@ int server_cli(int argc, char **argv)}// API key is invalid or not provided- res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8");- res.status = 401; // Unauthorized+ // TODO: make another middleware for CORS related logic+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ res_error(res, format_error_response("Invalid API Key", ERROR_TYPE_AUTHENTICATION));LOG_WARNING("Unauthorized: Invalid API Key", {});return false;};- svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- json data = {- { "user_name", llama.name_user.c_str() },- { "assistant_name", llama.name_assistant.c_str() },- { "default_generation_settings", llama.default_generation_settings_for_props },- { "total_slots", llama.params.n_parallel }- };- res.set_content(data.dump(), "application/json; charset=utf-8");- });+ // register server middlewares+ svr->set_pre_routing_handler([&middleware_validate_api_key](const httplib::Request & req, httplib::Response & res) {+ if (!middleware_validate_api_key(req, res)) {+ return httplib::Server::HandlerResponse::Handled;+ }+ return httplib::Server::HandlerResponse::Unhandled;+ });- svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- if (!validate_api_key(req, res)) {- return;- }- json data = json::parse(req.body);- const int task_id = llama.queue_tasks.get_new_id();- llama.queue_results.add_waiting_task_id(task_id);- llama.request_completion(task_id, data, false, false, -1);- if (!json_value(data, "stream", false)) {- std::string completion_text;- task_result result = llama.queue_results.recv(task_id);- if (!result.error && result.stop) {- res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");- }- else- {- res.status = 404;- res.set_content(result.result_json["content"], "text/plain; charset=utf-8");- }- llama.queue_results.remove_waiting_task_id(task_id);- } else {- const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink)- {- while (true)- {- task_result result = llama.queue_results.recv(task_id);- if (!result.error) {- const std::string str =- "data: " +- result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";- LOG_VERBOSE("data stream", {- { "to_send", str }- });- if (!sink.write(str.c_str(), str.size()))- {- llama.queue_results.remove_waiting_task_id(task_id);- return false;- }- if (result.stop) {- break;- }- } else {- const std::string str =- "error: " +- result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";- LOG_VERBOSE("data stream", {- { "to_send", str }- });- if (!sink.write(str.c_str(), str.size()))- {- llama.queue_results.remove_waiting_task_id(task_id);- return false;- }- break;- }- }+ //+ // Route handlers (or controllers)+ //- llama.queue_results.remove_waiting_task_id(task_id);- sink.done();- return true;- };+ const auto handle_health = [&](const httplib::Request & req, httplib::Response & res) {+ server_state current_state = state.load();+ switch (current_state) {+ case SERVER_STATE_READY:+ {+ // request slots data using task queue+ server_task task;+ task.id = ctx_server.queue_tasks.get_new_id();+ task.type = SERVER_TASK_TYPE_METRICS;+ task.id_target = -1;++ ctx_server.queue_results.add_waiting_task_id(task.id);+ ctx_server.queue_tasks.post(task);- auto on_complete = [task_id, &llama] (bool)- {- // cancel- llama.request_cancel(task_id);- llama.queue_results.remove_waiting_task_id(task_id);+ // get the result+ server_task_result result = ctx_server.queue_results.recv(task.id);+ ctx_server.queue_results.remove_waiting_task_id(task.id);++ const int n_idle_slots = result.data["idle"];+ const int n_processing_slots = result.data["processing"];++ json health = {+ {"status", "ok"},+ {"slots_idle", n_idle_slots},+ {"slots_processing", n_processing_slots}};- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);+ res.status = 200; // HTTP OK+ if (sparams.slots_endpoint && req.has_param("include_slots")) {+ health["slots"] = result.data["slots"];+ }++ if (n_idle_slots == 0) {+ health["status"] = "no slot available";+ if (req.has_param("fail_on_no_slot")) {+ res.status = 503; // HTTP Service Unavailable+ }+ }++ res.set_content(health.dump(), "application/json");+ break;}- });+ case SERVER_STATE_LOADING_MODEL:+ {+ res_error(res, format_error_response("Loading model", ERROR_TYPE_UNAVAILABLE));+ } break;+ case SERVER_STATE_ERROR:+ {+ res_error(res, format_error_response("Model failed to load", ERROR_TYPE_SERVER));+ } break;+ }+ };- svr.Get("/v1/models", [¶ms](const httplib::Request& req, httplib::Response& res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- std::time_t t = std::time(0);-- json models = {- {"object", "list"},- {"data", {- {- {"id", params.model_alias},- {"object", "model"},- {"created", t},- {"owned_by", "llamacpp"}- },- }}- };+ const auto handle_slots = [&](const httplib::Request &, httplib::Response & res) {+ if (!sparams.slots_endpoint) {+ res_error(res, format_error_response("This server does not support slots endpoint.", ERROR_TYPE_NOT_SUPPORTED));+ return;+ }- res.set_content(models.dump(), "application/json; charset=utf-8");- });+ // request slots data using task queue+ server_task task;+ task.id = ctx_server.queue_tasks.get_new_id();+ task.id_multi = -1;+ task.id_target = -1;+ task.type = SERVER_TASK_TYPE_METRICS;+ ctx_server.queue_results.add_waiting_task_id(task.id);+ ctx_server.queue_tasks.post(task);- // TODO: add mount point without "/v1" prefix -- how?- svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- if (!validate_api_key(req, res)) {- return;- }- json data = oaicompat_completion_params_parse(llama.model, json::parse(req.body), sparams.chat_template);+ // get the result+ server_task_result result = ctx_server.queue_results.recv(task.id);+ ctx_server.queue_results.remove_waiting_task_id(task.id);++ res.set_content(result.data["slots"].dump(), "application/json");+ res.status = 200; // HTTP OK+ };++ const auto handle_metrics = [&](const httplib::Request &, httplib::Response & res) {+ if (!sparams.metrics_endpoint) {+ res_error(res, format_error_response("This server does not support metrics endpoint.", ERROR_TYPE_NOT_SUPPORTED));+ return;+ }++ // request slots data using task queue+ server_task task;+ task.id = ctx_server.queue_tasks.get_new_id();+ task.id_multi = -1;+ task.id_target = -1;+ task.type = SERVER_TASK_TYPE_METRICS;+ task.data.push_back({{"reset_bucket", true}});++ ctx_server.queue_results.add_waiting_task_id(task.id);+ ctx_server.queue_tasks.post(task);++ // get the result+ server_task_result result = ctx_server.queue_results.recv(task.id);+ ctx_server.queue_results.remove_waiting_task_id(task.id);++ json data = result.data;++ const uint64_t n_prompt_tokens_processed = data["n_prompt_tokens_processed"];+ const uint64_t t_prompt_processing = data["t_prompt_processing"];++ const uint64_t n_tokens_predicted = data["n_tokens_predicted"];+ const uint64_t t_tokens_generation = data["t_tokens_generation"];++ const int32_t kv_cache_used_cells = data["kv_cache_used_cells"];++ // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names+ json all_metrics_def = json {+ {"counter", {{+ {"name", "prompt_tokens_total"},+ {"help", "Number of prompt tokens processed."},+ {"value", (uint64_t) data["n_prompt_tokens_processed_total"]}+ }, {+ {"name", "prompt_seconds_total"},+ {"help", "Prompt process time"},+ {"value", (uint64_t) data["t_prompt_processing_total"] / 1.e3}+ }, {+ {"name", "tokens_predicted_total"},+ {"help", "Number of generation tokens processed."},+ {"value", (uint64_t) data["n_tokens_predicted_total"]}+ }, {+ {"name", "tokens_predicted_seconds_total"},+ {"help", "Predict process time"},+ {"value", (uint64_t) data["t_tokens_generation_total"] / 1.e3}+ }}},+ {"gauge", {{+ {"name", "prompt_tokens_seconds"},+ {"help", "Average prompt throughput in tokens/s."},+ {"value", n_prompt_tokens_processed ? 1.e3 / t_prompt_processing * n_prompt_tokens_processed : 0.}+ },{+ {"name", "predicted_tokens_seconds"},+ {"help", "Average generation throughput in tokens/s."},+ {"value", n_tokens_predicted ? 1.e3 / t_tokens_generation * n_tokens_predicted : 0.}+ },{+ {"name", "kv_cache_usage_ratio"},+ {"help", "KV-cache usage. 1 means 100 percent usage."},+ {"value", 1. * kv_cache_used_cells / params.n_ctx}+ },{+ {"name", "kv_cache_tokens"},+ {"help", "KV-cache tokens."},+ {"value", (uint64_t) data["kv_cache_tokens_count"]}+ },{+ {"name", "requests_processing"},+ {"help", "Number of request processing."},+ {"value", (uint64_t) data["processing"]}+ },{+ {"name", "requests_deferred"},+ {"help", "Number of request deferred."},+ {"value", (uint64_t) data["deferred"]}+ }}}+ };++ std::stringstream prometheus;++ for (const auto & el : all_metrics_def.items()) {+ const auto & type = el.key();+ const auto & metrics_def = el.value();++ for (const auto & metric_def : metrics_def) {+ const std::string name = metric_def["name"];+ const std::string help = metric_def["help"];- const int task_id = llama.queue_tasks.get_new_id();- llama.queue_results.add_waiting_task_id(task_id);- llama.request_completion(task_id, data, false, false, -1);+ auto value = json_value(metric_def, "value", 0.);+ prometheus << "# HELP llamacpp:" << name << " " << help << "\n"+ << "# TYPE llamacpp:" << name << " " << type << "\n"+ << "llamacpp:" << name << " " << value << "\n";+ }+ }++ const int64_t t_start = data["t_start"];+ res.set_header("Process-Start-Time-Unix", std::to_string(t_start));++ res.set_content(prometheus.str(), "text/plain; version=0.0.4");+ res.status = 200; // HTTP OK+ };++ const auto handle_props = [&ctx_server](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ json data = {+ { "user_name", ctx_server.name_user.c_str() },+ { "assistant_name", ctx_server.name_assistant.c_str() },+ { "default_generation_settings", ctx_server.default_generation_settings_for_props },+ { "total_slots", ctx_server.params.n_parallel }+ };++ res.set_content(data.dump(), "application/json; charset=utf-8");+ };++ const auto handle_completions = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));++ json data = json::parse(req.body);- if (!json_value(data, "stream", false)) {- std::string completion_text;- task_result result = llama.queue_results.recv(task_id);+ const int id_task = ctx_server.queue_tasks.get_new_id();- if (!result.error && result.stop) {- json oaicompat_result = format_final_response_oaicompat(data, result);+ ctx_server.queue_results.add_waiting_task_id(id_task);+ ctx_server.request_completion(id_task, -1, data, false, false);- res.set_content(oaicompat_result.dump(-1, ' ', false,- json::error_handler_t::replace),- "application/json; charset=utf-8");+ if (!json_value(data, "stream", false)) {+ server_task_result result = ctx_server.queue_results.recv(id_task);+ if (!result.error && result.stop) {+ res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");+ } else {+ res_error(res, result.data);+ }++ ctx_server.queue_results.remove_waiting_task_id(id_task);+ } else {+ const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) {+ while (true) {+ server_task_result result = ctx_server.queue_results.recv(id_task);+ if (!result.error) {+ const std::string str =+ "data: " ++ result.data.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";++ LOG_VERBOSE("data stream", {+ { "to_send", str }+ });++ if (!sink.write(str.c_str(), str.size())) {+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ return false;+ }++ if (result.stop) {+ break;+ }} else {- res.status = 500;- res.set_content(result.result_json["content"], "text/plain; charset=utf-8");+ const std::string str =+ "error: " ++ result.data.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";++ LOG_VERBOSE("data stream", {+ { "to_send", str }+ });++ if (!sink.write(str.c_str(), str.size())) {+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ return false;+ }++ break;}- llama.queue_results.remove_waiting_task_id(task_id);- } else {- const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) {- while (true) {- task_result llama_result = llama.queue_results.recv(task_id);- if (!llama_result.error) {- std::vector<json> result_array = format_partial_response_oaicompat( llama_result);-- for (auto it = result_array.begin(); it != result_array.end(); ++it)- {- if (!it->empty()) {- const std::string str =- "data: " +- it->dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";- LOG_VERBOSE("data stream", {{"to_send", str}});- if (!sink.write(str.c_str(), str.size())) {- llama.queue_results.remove_waiting_task_id(task_id);- return false;- }- }- }- if (llama_result.stop) {- break;- }- } else {+ }++ ctx_server.queue_results.remove_waiting_task_id(id_task);+ sink.done();++ return true;+ };++ auto on_complete = [id_task, &ctx_server] (bool) {+ // cancel+ ctx_server.request_cancel(id_task);+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ };++ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);+ }+ };++ const auto handle_models = [¶ms, &model_meta](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));++ json models = {+ {"object", "list"},+ {"data", {+ {+ {"id", params.model_alias},+ {"object", "model"},+ {"created", std::time(0)},+ {"owned_by", "llamacpp"},+ {"meta", model_meta}+ },+ }}+ };++ res.set_content(models.dump(), "application/json; charset=utf-8");+ };++ const auto handle_chat_completions = [&ctx_server, &sparams, &res_error](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), sparams.chat_template);++ const int id_task = ctx_server.queue_tasks.get_new_id();++ ctx_server.queue_results.add_waiting_task_id(id_task);+ ctx_server.request_completion(id_task, -1, data, false, false);++ const auto completion_id = gen_chatcmplid();+ if (!json_value(data, "stream", false)) {+ server_task_result result = ctx_server.queue_results.recv(id_task);++ if (!result.error && result.stop) {+ json result_oai = format_final_response_oaicompat(data, result.data, completion_id);++ res.set_content(result_oai.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");+ } else {+ res_error(res, result.data);+ }+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ } else {+ const auto chunked_content_provider = [id_task, &ctx_server, completion_id](size_t, httplib::DataSink & sink) {+ while (true) {+ server_task_result result = ctx_server.queue_results.recv(id_task);+ if (!result.error) {+ std::vector<json> result_array = format_partial_response_oaicompat(result.data, completion_id);++ for (auto it = result_array.begin(); it != result_array.end(); ++it) {+ if (!it->empty()) {const std::string str =- "error: " +- llama_result.result_json.dump(-1, ' ', false,- json::error_handler_t::replace) ++ "data: " ++ it->dump(-1, ' ', false, json::error_handler_t::replace) +"\n\n";LOG_VERBOSE("data stream", {{"to_send", str}});if (!sink.write(str.c_str(), str.size())) {- llama.queue_results.remove_waiting_task_id(task_id);+ ctx_server.queue_results.remove_waiting_task_id(id_task);return false;}- break;}}- sink.done();- llama.queue_results.remove_waiting_task_id(task_id);- return true;- };+ if (result.stop) {+ break;+ }+ } else {+ const std::string str =+ "error: " ++ result.data.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";+ LOG_VERBOSE("data stream", {{"to_send", str}});+ if (!sink.write(str.c_str(), str.size())) {+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ return false;+ }+ break;+ }+ }+ sink.done();+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ return true;+ };- auto on_complete = [task_id, &llama](bool) {- // cancel request- llama.request_cancel(task_id);- llama.queue_results.remove_waiting_task_id(task_id);- };+ auto on_complete = [id_task, &ctx_server](bool) {+ // cancel request+ ctx_server.request_cancel(id_task);+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ };- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);- }- });+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);+ }+ };- svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- if (!validate_api_key(req, res)) {- return;- }- json data = json::parse(req.body);- const int task_id = llama.queue_tasks.get_new_id();- llama.queue_results.add_waiting_task_id(task_id);- llama.request_completion(task_id, data, true, false, -1);- if (!json_value(data, "stream", false)) {- std::string completion_text;- task_result result = llama.queue_results.recv(task_id);- if (!result.error && result.stop)- {- res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");- }- else- {- res.status = 404;- res.set_content(result.result_json["content"], "text/plain; charset=utf-8");- }- llama.queue_results.remove_waiting_task_id(task_id);- } else {- const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink) {- while (true)- {- task_result result = llama.queue_results.recv(task_id);- if (!result.error) {- const std::string str =- "data: " +- result.result_json.dump(-1, ' ', false, json::error_handler_t::replace) +- "\n\n";- LOG_VERBOSE("data stream", {- { "to_send", str }- });- if (!sink.write(str.c_str(), str.size()))- {- llama.queue_results.remove_waiting_task_id(task_id);- return false;- }- if (result.stop)- {- break;- }- }- else- {- break;- }- }+ const auto handle_infill = [&ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- llama.queue_results.remove_waiting_task_id(task_id);- sink.done();- return true;- };+ json data = json::parse(req.body);- auto on_complete = [task_id, &llama] (bool)- {- // cancel- llama.request_cancel(task_id);- };+ const int id_task = ctx_server.queue_tasks.get_new_id();- res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);- }- });+ ctx_server.queue_results.add_waiting_task_id(id_task);+ ctx_server.request_completion(id_task, -1, data, true, false);- svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res)- { return res.set_content("", "application/json; charset=utf-8"); });+ if (!json_value(data, "stream", false)) {+ server_task_result result = ctx_server.queue_results.recv(id_task);+ if (!result.error && result.stop) {+ res.set_content(result.data.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");+ } else {+ res_error(res, result.data);+ }- svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- const json body = json::parse(req.body);- std::vector<llama_token> tokens;- if (body.count("content") != 0)- {- tokens = llama.tokenize(body["content"], false);- }- const json data = format_tokenizer_response(tokens);- return res.set_content(data.dump(), "application/json; charset=utf-8");- });+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ } else {+ const auto chunked_content_provider = [id_task, &ctx_server](size_t, httplib::DataSink & sink) {+ while (true) {+ server_task_result result = ctx_server.queue_results.recv(id_task);+ if (!result.error) {+ const std::string str =+ "data: " ++ result.data.dump(-1, ' ', false, json::error_handler_t::replace) ++ "\n\n";++ LOG_VERBOSE("data stream", {+ { "to_send", str }+ });- svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- const json body = json::parse(req.body);- std::string content;- if (body.count("tokens") != 0)- {- const std::vector<llama_token> tokens = body["tokens"];- content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());+ if (!sink.write(str.c_str(), str.size())) {+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ return false;+ }++ if (result.stop) {+ break;+ }+ } else {+ break;+ }}- const json data = format_detokenized_response(content);- return res.set_content(data.dump(), "application/json; charset=utf-8");- });+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ sink.done();- svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- const json body = json::parse(req.body);- json prompt;- if (body.count("content") != 0)- {- prompt = body["content"];- }- else- {- prompt = "";- }+ return true;+ };- json image_data;- if (body.count("image_data") != 0) {- image_data = body["image_data"];- }- else- {- image_data = "";- }+ auto on_complete = [id_task, &ctx_server] (bool) {+ ctx_server.request_cancel(id_task);+ };- // create and queue the task- const int task_id = llama.queue_tasks.get_new_id();- llama.queue_results.add_waiting_task_id(task_id);- llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1);+ res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);+ }+ };- // get the result- task_result result = llama.queue_results.recv(task_id);- llama.queue_results.remove_waiting_task_id(task_id);+ const auto handle_tokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ const json body = json::parse(req.body);- // send the result- return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");- });+ std::vector<llama_token> tokens;+ if (body.count("content") != 0) {+ tokens = ctx_server.tokenize(body["content"], false);+ }+ const json data = format_tokenizer_response(tokens);+ return res.set_content(data.dump(), "application/json; charset=utf-8");+ };- svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res)- {- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));- const json body = json::parse(req.body);+ const auto handle_detokenize = [&ctx_server](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ const json body = json::parse(req.body);- json prompt;- if (body.count("input") != 0)- {- prompt = body["input"];- // batch- if(prompt.is_array()) {- json data = json::array();- int i = 0;- for (const json &elem : prompt) {- const int task_id = llama.queue_tasks.get_new_id();- llama.queue_results.add_waiting_task_id(task_id);- llama.request_completion(task_id, { {"prompt", elem}, { "n_predict", 0} }, false, true, -1);-- // get the result- task_result result = llama.queue_results.recv(task_id);- llama.queue_results.remove_waiting_task_id(task_id);-- json embedding = json{- {"embedding", json_value(result.result_json, "embedding", json::array())},- {"index", i++},- {"object", "embedding"}- };- data.push_back(embedding);- }- json result = format_embeddings_response_oaicompat(body, data);- return res.set_content(result.dump(), "application/json; charset=utf-8");- }- }- else- {- prompt = "";+ std::string content;+ if (body.count("tokens") != 0) {+ const std::vector<llama_token> tokens = body["tokens"];+ content = tokens_to_str(ctx_server.ctx, tokens.cbegin(), tokens.cend());+ }++ const json data = format_detokenized_response(content);+ return res.set_content(data.dump(), "application/json; charset=utf-8");+ };++ const auto handle_embeddings = [¶ms, &ctx_server, &res_error](const httplib::Request & req, httplib::Response & res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ if (!params.embedding) {+ res.status = 501;+ res.set_content("This server does not support embeddings. Start it with `--embeddings`", "text/plain; charset=utf-8");+ return;+ }++ const json body = json::parse(req.body);+ bool is_openai = false;++ // an input prompt can be a string or a list of tokens (integer)+ json prompt;+ if (body.count("input") != 0) {+ is_openai = true;+ prompt = body["input"];+ } else if (body.count("content") != 0) {+ // with "content", we only support single prompt+ prompt = std::vector<std::string>{body["content"]};+ } else {+ res_error(res, format_error_response("\"input\" or \"content\" must be provided", ERROR_TYPE_INVALID_REQUEST));+ return;+ }++ // create and queue the task+ json responses;+ {+ const int id_task = ctx_server.queue_tasks.get_new_id();+ ctx_server.queue_results.add_waiting_task_id(id_task);+ ctx_server.request_completion(id_task, -1, {{"prompt", prompt}}, false, true);++ // get the result+ server_task_result result = ctx_server.queue_results.recv(id_task);+ ctx_server.queue_results.remove_waiting_task_id(id_task);+ if (!result.error) {+ if (result.data.count("results")) {+ // result for multi-task+ responses = result.data["results"];+ } else {+ // result for single task+ responses = std::vector<json>{result.data};}+ } else {+ // error received, ignore everything else+ res_error(res, result.data);+ return;+ }+ }++ // write JSON response+ json root = is_openai+ ? format_embeddings_response_oaicompat(body, responses)+ : responses[0];+ return res.set_content(root.dump(), "application/json; charset=utf-8");+ };++ auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) {+ return [content, len, mime_type](const httplib::Request &, httplib::Response & res) {+ res.set_content(reinterpret_cast<const char*>(content), len, mime_type);+ return false;+ };+ };- // create and queue the task- const int task_id = llama.queue_tasks.get_new_id();- llama.queue_results.add_waiting_task_id(task_id);- llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}}, false, true, -1);+ //+ // Router+ //- // get the result- task_result result = llama.queue_results.recv(task_id);- llama.queue_results.remove_waiting_task_id(task_id);+ // register static assets routes+ if (!sparams.public_path.empty()) {+ // Set the base directory for serving static files+ svr->set_base_dir(sparams.public_path);+ }- json data = json::array({json{- {"embedding", json_value(result.result_json, "embedding", json::array())},- {"index", 0},- {"object", "embedding"}- }}- );+ // register API routes+ svr->Get ("/health", handle_health);+ svr->Get ("/slots", handle_slots);+ svr->Get ("/metrics", handle_metrics);+ svr->Get ("/props", handle_props);+ svr->Get ("/v1/models", handle_models);+ svr->Post("/completion", handle_completions); // legacy+ svr->Post("/completions", handle_completions);+ svr->Post("/v1/completions", handle_completions);+ svr->Post("/chat/completions", handle_chat_completions);+ svr->Post("/v1/chat/completions", handle_chat_completions);+ svr->Post("/infill", handle_infill);+ svr->Post("/embedding", handle_embeddings); // legacy+ svr->Post("/embeddings", handle_embeddings);+ svr->Post("/v1/embeddings", handle_embeddings);+ svr->Post("/tokenize", handle_tokenize);+ svr->Post("/detokenize", handle_detokenize);++ //+ // Start the server+ //+ if (sparams.n_threads_http < 1) {+ // +2 threads for monitoring endpoints+ sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1);+ }+ log_data["n_threads_http"] = std::to_string(sparams.n_threads_http);+ svr->new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); };- json root = format_embeddings_response_oaicompat(body, data);+ LOG_INFO("HTTP server listening", log_data);- // send the result- return res.set_content(root.dump(), "application/json; charset=utf-8");- });+ // launch browser tab+ if (!sparams.nobrowser) {+ char url[128];+ snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);+ llamafile_launch_browser(url);+ }- // GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!?- // "Bus error: 10" - this is on macOS, it does not crash on Linux- //std::thread t2([&]()- /*{- bool running = true;- while (running)- {- running = llama.update_slots();- }- }*/- //);-- llama.queue_tasks.on_new_task(std::bind(- &llama_server_context::process_single_task, &llama, std::placeholders::_1));- llama.queue_tasks.on_finish_multitask(std::bind(- &llama_server_context::on_finish_multitask, &llama, std::placeholders::_1));- llama.queue_tasks.on_all_tasks_finished(std::bind(- &llama_server_context::run_on_all_tasks_finished, &llama));- llama.queue_results.on_multitask_update(std::bind(- &llama_server_queue::update_multitask,- &llama.queue_tasks,+ if (!FLAG_unsecure) {+ if (IsXnu()) {+ // Cosmopolitan libc explicitly does not support cosmo_dlopen on x64+ // macOS and mac_sandbox_init depends on cosmo_dlopen. We'll attempt+ // to enable sandboxing and ignore the failure on x64 macOS.+ // (Alternatively, the sandbox-exec(1) command could be used to+ // launch llamafile with a sandbox profile.) A goal of llamafile is+ // to work easily out-of-the-box and it already operates without a+ // sandbox on macOS so don't require --unsecure on x64 macOS to+ // workaround the failure. If support is added later, it will be+ // used without further changes. On Apple Silicon, if sandbox+ // initialization fails, exit and recommend using --unsecure to+ // disable sandboxing.+ std::string error_msg;+ int rv = mac_sandbox_init(error_msg);+ if (rv != 0) {+ fprintf(stderr, "Sandbox initialization failed:\n"+ "rv: %d, error: %s.\n"+ "Disable sandboxing with the --unsecure option.\n",+ rv, error_msg.c_str());+ if (IsXnuSilicon()) {+ exit(1);+ } else {+ LOG_WARNING("Sandbox initialization failed. Not supported "+ "on x64 macOS. Continuing.", {});+ }+ }+ } else if (!llamafile_has_gpu()) {+ printf("In the sandboxing block!\n");+ // Enables pledge() security on Linux and OpenBSD.+ // - We do this *after* binding the server socket.+ // - We do this *after* opening the log file for writing.+ // - We do this *after* opening a tab in the user browser.+ // - We do this *before* loading any weights or graphdefs.+ // In effect, what this does is:+ // - Filesystem access is disabled entirely (except ZipOS).+ // - On Linux, network access is restricted to accept() only.+ // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.+ char promises[32];+ if (IsOpenbsd()) {+ strlcpy(promises, "stdio inet", sizeof(promises));+ } else {+ strlcpy(promises, "stdio anet", sizeof(promises));+ }+ if (!startswith(sparams.public_path.c_str(), "/zip/")) {+ strlcat(promises, " rpath", sizeof(promises));+ }+ __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;+ if (pledge(0, 0)) {+ LOG_TEE("warning: this OS doesn't support pledge() security\n");+ } else if (pledge(promises, 0)) {+ perror("pledge");+ exit(1);+ }+ }+ }++ // run the HTTP server in a thread - see comment below+ std::thread t([&]() {+ if (!svr->listen_after_bind()) {+ state.store(SERVER_STATE_ERROR);+ return 1;+ }++ return 0;+ });++ ctx_server.queue_tasks.on_new_task(std::bind(+ &server_context::process_single_task, &ctx_server, std::placeholders::_1));+ ctx_server.queue_tasks.on_finish_multitask(std::bind(+ &server_context::on_finish_multitask, &ctx_server, std::placeholders::_1));+ ctx_server.queue_tasks.on_update_slots(std::bind(+ &server_context::update_slots, &ctx_server));+ ctx_server.queue_results.on_multitask_update(std::bind(+ &server_queue::update_multitask,+ &ctx_server.queue_tasks,std::placeholders::_1,std::placeholders::_2,std::placeholders::_3));shutdown_handler = [&](int) {- llama.queue_tasks.terminate();+ ctx_server.queue_tasks.terminate();};#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))@@ -3639,10 +3669,13 @@ int server_cli(int argc, char **argv)};SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);#endif- llama.queue_tasks.start_loop();- svr.stop();++ ctx_server.queue_tasks.start_loop();++ svr->stop();t.join();llama_backend_free();+return 0;}diff --git a/llama.cpp/server/utils.h b/llama.cpp/server/utils.hindex 095c2fa2..b84cf70e 100644--- a/llama.cpp/server/utils.h+++ b/llama.cpp/server/utils.h@@ -1,21 +1,31 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#pragma once+#include "llama.cpp/llama.h"+#include "llama.cpp/common.h"+#include "llama.cpp/json.h"+#include <string>#include <vector>-#include <set>-#include <mutex>-#include <condition_variable>-#include <unordered_map>+#include <sstream>+#include <random>-#include "json.h"--#include "llama.cpp/llava/clip.h"+#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"using json = nlohmann::json;+// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/11+enum error_type {+ ERROR_TYPE_INVALID_REQUEST,+ ERROR_TYPE_AUTHENTICATION,+ ERROR_TYPE_SERVER,+ ERROR_TYPE_NOT_FOUND,+ ERROR_TYPE_PERMISSION,+ ERROR_TYPE_UNAVAILABLE, // custom error+ ERROR_TYPE_NOT_SUPPORTED, // custom error+};+extern bool server_verbose;extern bool server_log_json;@@ -40,125 +50,35 @@ extern bool server_log_json;#define LOG_WARNING(MSG, ...) server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__)#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__)-//-// parallel-//--enum server_state {- SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet- SERVER_STATE_READY, // Server is ready and model is loaded- SERVER_STATE_ERROR // An error occurred, load_model failed-};--enum task_type {- TASK_TYPE_COMPLETION,- TASK_TYPE_CANCEL,- TASK_TYPE_NEXT_RESPONSE,- TASK_TYPE_METRICS-};--struct task_server {- int id = -1; // to be filled by llama_server_queue- int target_id;- task_type type;- json data;- bool infill_mode = false;- bool embedding_mode = false;- int multitask_id = -1;-};--struct task_result {- int id;- int multitask_id = -1;- bool stop;- bool error;- json result_json;-};--struct task_multi {- int id;- std::set<int> subtasks_remaining{};- std::vector<task_result> results{};-};--// TODO: can become bool if we can't find use of more states-enum slot_state-{- IDLE,- PROCESSING,-};--enum slot_command-{- NONE,- LOAD_PROMPT,- RELEASE,-};--struct slot_params-{- bool stream = true;- bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt-- uint32_t seed = -1; // RNG seed- int32_t n_keep = 0; // number of tokens to keep from initial prompt- int32_t n_predict = -1; // new tokens to predict-- std::vector<std::string> antiprompt;-- json input_prefix;- json input_suffix;-};--struct slot_image-{- int32_t id;-- bool request_encode_image = false;- float * image_embedding = nullptr;- int32_t image_tokens = 0;-- clip_image_u8 * img_data;-- std::string prefix_prompt; // before of this image-};--// completion token output with probabilities-struct completion_token_output-{- struct token_prob- {- llama_token tok;- float prob;- };-- std::vector<token_prob> probs;- llama_token tok;- std::string text_to_send;-};+template <typename T>+static T json_value(const json &body, const std::string &key, const T &default_value) {+ // Fallback null to default value+ return body.contains(key) && !body.at(key).is_null()+ ? body.value(key, default_value)+ : default_value;+}-static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra)-{+static inline void server_log(const char *level, const char *function, int line, const char *message, const nlohmann::ordered_json &extra) {std::stringstream ss_tid;ss_tid << std::this_thread::get_id();json log = nlohmann::ordered_json{- {"tid", ss_tid.str()},+ {"tid", ss_tid.str()},{"timestamp", time(nullptr)},};if (server_log_json) {- log.merge_patch(- {- {"level", level},- {"function", function},- {"line", line},- {"msg", message},- });+ log.merge_patch( {+ {"level", level},+ {"function", function},+ {"line", line},+ {"msg", message},+ });+if (!extra.empty()) {log.merge_patch(extra);}- std::cout << log.dump(-1, ' ', false, json::error_handler_t::replace) << "\n" << std::flush;+ printf("%s\n", log.dump(-1, ' ', false, json::error_handler_t::replace).c_str());} else {char buf[1024];snprintf(buf, 1024, "%4s [%24s] %s", level, function, message);@@ -171,8 +91,7 @@ static inline void server_log(const char *level, const char *function, int line,for (const auto& el : log.items()){const std::string value = el.value().dump(-1, ' ', false, json::error_handler_t::replace);- snprintf(buf, 1024, " %s=%s", el.key().c_str(), value.c_str());- ss << buf;+ ss << " " << el.key() << "=" << value;}const std::string str = ss.str();@@ -182,36 +101,25 @@ static inline void server_log(const char *level, const char *function, int line,}//-// server utils+// chat template utils//-template <typename T>-static T json_value(const json &body, const std::string &key, const T &default_value)-{- // Fallback null to default value- return body.contains(key) && !body.at(key).is_null()- ? body.value(key, default_value)- : default_value;-}-// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's validinline bool verify_custom_template(const std::string & tmpl) {llama_chat_message chat[] = {{"user", "test"}};- std::vector<char> buf(1);- int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, buf.data(), buf.size());+ int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, nullptr, 0);return res >= 0;}// Format given chat. If tmpl is empty, we take the template from model metadata-inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages)-{+inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector<json> & messages) {size_t alloc_size = 0;// vector holding all allocated string to be passed to llama_chat_apply_templatestd::vector<std::string> str(messages.size() * 2);std::vector<llama_chat_message> chat(messages.size());for (size_t i = 0; i < messages.size(); ++i) {- auto &curr_msg = messages[i];+ const auto & curr_msg = messages[i];str[i*2 + 0] = json_value(curr_msg, "role", std::string(""));str[i*2 + 1] = json_value(curr_msg, "content", std::string(""));alloc_size += str[i*2 + 1].length();@@ -231,252 +139,13 @@ inline std::string format_chat(const struct llama_model * model, const std::strires = llama_chat_apply_template(model, ptr_tmpl, chat.data(), chat.size(), true, buf.data(), buf.size());}- std::string formatted_chat(buf.data(), res);+ const std::string formatted_chat(buf.data(), res);+LOG_VERBOSE("formatted_chat", {{"text", formatted_chat.c_str()}});return formatted_chat;}-//-// work queue utils-//--struct llama_server_queue {- int id = 0;- std::mutex mutex_tasks;- bool running;- // queues- std::vector<task_server> queue_tasks;- std::vector<task_server> queue_tasks_deferred;- std::vector<task_multi> queue_multitasks;- std::condition_variable condition_tasks;- // callback functions- std::function<void(task_server&)> callback_new_task;- std::function<void(task_multi&)> callback_finish_multitask;- std::function<void(void)> callback_all_task_finished;-- // Add a new task to the end of the queue- int post(task_server task) {- std::unique_lock<std::mutex> lock(mutex_tasks);- if (task.id == -1) {- task.id = id++;- LOG_VERBOSE("new task id", {{"new_id", task.id}});- }- queue_tasks.push_back(std::move(task));- condition_tasks.notify_one();- return task.id;- }-- // Add a new task, but defer until one slot is available- void defer(task_server task) {- std::unique_lock<std::mutex> lock(mutex_tasks);- queue_tasks_deferred.push_back(std::move(task));- }-- // Get the next id for creating anew task- int get_new_id() {- std::unique_lock<std::mutex> lock(mutex_tasks);- int new_id = id++;- LOG_VERBOSE("new task id", {{"new_id", new_id}});- return new_id;- }-- // Register function to process a new task- void on_new_task(std::function<void(task_server&)> callback) {- callback_new_task = callback;- }-- // Register function to process a multitask- void on_finish_multitask(std::function<void(task_multi&)> callback) {- callback_finish_multitask = callback;- }-- // Register the function to be called when the batch of tasks is finished- void on_all_tasks_finished(std::function<void(void)> callback) {- callback_all_task_finished = callback;- }-- // Call when the state of one slot is changed- void notify_slot_changed() {- // move deferred tasks back to main loop- std::unique_lock<std::mutex> lock(mutex_tasks);- for (auto & task : queue_tasks_deferred) {- queue_tasks.push_back(std::move(task));- }- queue_tasks_deferred.clear();- }-- // end the start_loop routine- void terminate() {- {- std::unique_lock<std::mutex> lock(mutex_tasks);- running = false;- }- condition_tasks.notify_all();- }-- // Start the main loop.- void start_loop() {- running = true;- while (true) {- LOG_VERBOSE("new task may arrive", {});- {- while (true)- {- std::unique_lock<std::mutex> lock(mutex_tasks);- if (queue_tasks.empty()) {- lock.unlock();- break;- }- task_server task = queue_tasks.front();- queue_tasks.erase(queue_tasks.begin());- lock.unlock();- LOG_VERBOSE("callback_new_task", {{"task_id", task.id}});- callback_new_task(task);- }- LOG_VERBOSE("callback_all_task_finished", {});- // process and update all the multitasks- auto queue_iterator = queue_multitasks.begin();- while (queue_iterator != queue_multitasks.end())- {- if (queue_iterator->subtasks_remaining.empty())- {- // all subtasks done == multitask is done- task_multi current_multitask = *queue_iterator;- callback_finish_multitask(current_multitask);- // remove this multitask- queue_iterator = queue_multitasks.erase(queue_iterator);- }- else- {- ++queue_iterator;- }- }- // all tasks in the current loop is finished- callback_all_task_finished();- }- LOG_VERBOSE("wait for new task", {});- // wait for new task- {- std::unique_lock<std::mutex> lock(mutex_tasks);- if (queue_tasks.empty()) {- if (!running) {- LOG_VERBOSE("ending start_loop", {});- return;- }- condition_tasks.wait(lock, [&]{- return (!queue_tasks.empty() || !running);- });- }- }- }- }-- //- // functions to manage multitasks- //-- // add a multitask by specifying the id of all subtask (subtask is a task_server)- void add_multitask(int multitask_id, std::vector<int>& sub_ids)- {- std::lock_guard<std::mutex> lock(mutex_tasks);- task_multi multi;- multi.id = multitask_id;- std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));- queue_multitasks.push_back(multi);- }-- // updatethe remaining subtasks, while appending results to multitask- void update_multitask(int multitask_id, int subtask_id, task_result& result)- {- std::lock_guard<std::mutex> lock(mutex_tasks);- for (auto& multitask : queue_multitasks)- {- if (multitask.id == multitask_id)- {- multitask.subtasks_remaining.erase(subtask_id);- multitask.results.push_back(result);- }- }- }-};--struct llama_server_response {- typedef std::function<void(int, int, task_result&)> callback_multitask_t;- callback_multitask_t callback_update_multitask;- // for keeping track of all tasks waiting for the result- std::set<int> waiting_task_ids;- // the main result queue- std::vector<task_result> queue_results;- std::mutex mutex_results;- std::condition_variable condition_results;-- void add_waiting_task_id(int task_id) {- LOG_VERBOSE("waiting for task id", {{"task_id", task_id}});- std::unique_lock<std::mutex> lock(mutex_results);- waiting_task_ids.insert(task_id);- }-- void remove_waiting_task_id(int task_id) {- LOG_VERBOSE("remove waiting for task id", {{"task_id", task_id}});- std::unique_lock<std::mutex> lock(mutex_results);- waiting_task_ids.erase(task_id);- }-- // This function blocks the thread until there is a response for this task_id- task_result recv(int task_id) {- while (true)- {- std::unique_lock<std::mutex> lock(mutex_results);- condition_results.wait(lock, [&]{- return !queue_results.empty();- });-- for (int i = 0; i < (int) queue_results.size(); i++)- {- if (queue_results[i].id == task_id)- {- assert(queue_results[i].multitask_id == -1);- task_result res = queue_results[i];- queue_results.erase(queue_results.begin() + i);- return res;- }- }- }-- // should never reach here- }-- // Register the function to update multitask- void on_multitask_update(callback_multitask_t callback) {- callback_update_multitask = callback;- }-- // Send a new result to a waiting task_id- void send(task_result result) {- std::unique_lock<std::mutex> lock(mutex_results);- LOG_VERBOSE("send new result", {{"task_id", result.id}});- for (auto& task_id : waiting_task_ids) {- // LOG_TEE("waiting task id %i \n", task_id);- // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result- if (result.multitask_id == task_id)- {- LOG_VERBOSE("callback_update_multitask", {{"task_id", task_id}});- callback_update_multitask(task_id, result.id, result);- continue;- }-- if (result.id == task_id)- {- LOG_VERBOSE("queue_results.push_back", {{"task_id", task_id}});- queue_results.push_back(result);- condition_results.notify_all();- return;- }- }- }-};-//// base64 utils (TODO: move to common in the future)//@@ -486,13 +155,11 @@ static const std::string base64_chars ="abcdefghijklmnopqrstuvwxyz""0123456789+/";-static inline bool is_base64(uint8_t c)-{+static inline bool is_base64(uint8_t c) {return (isalnum(c) || (c == '+') || (c == '/'));}-static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string)-{+static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string) {int i = 0;int j = 0;int in_ = 0;@@ -504,13 +171,10 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_strstd::vector<uint8_t> ret;- while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))- {+ while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {char_array_4[i++] = encoded_string[in_]; in_++;- if (i == 4)- {- for (i = 0; i <4; i++)- {+ if (i == 4) {+ for (i = 0; i < 4; i++) {char_array_4[i] = base64_chars.find(char_array_4[i]);}@@ -518,23 +182,20 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_strchar_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];- for (i = 0; (i < 3); i++)- {+ for (i = 0; (i < 3); i++) {ret.push_back(char_array_3[i]);}+i = 0;}}- if (i)- {- for (j = i; j <4; j++)- {+ if (i) {+ for (j = i; j < 4; j++) {char_array_4[j] = 0;}- for (j = 0; j <4; j++)- {+ for (j = 0; j < 4; j++) {char_array_4[j] = base64_chars.find(char_array_4[j]);}@@ -542,8 +203,7 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_strchar_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];- for (j = 0; (j < i - 1); j++)- {+ for (j = 0; j < i - 1; j++) {ret.push_back(char_array_3[j]);}}@@ -555,8 +215,7 @@ static inline std::vector<uint8_t> base64_decode(const std::string & encoded_str// random string / id//-static std::string random_string()-{+static std::string random_string() {static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");std::random_device rd;@@ -571,9 +230,389 @@ static std::string random_string()return result;}-static std::string gen_chatcmplid()-{+static std::string gen_chatcmplid() {std::stringstream chatcmplid;chatcmplid << "chatcmpl-" << random_string();+return chatcmplid.str();}++//+// other common utils+//++static size_t common_part(const std::vector<llama_token> & a, const std::vector<llama_token> & b) {+ size_t i;+ for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {}++ return i;+}++static bool ends_with(const std::string & str, const std::string & suffix) {+ return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);+}++static size_t find_partial_stop_string(const std::string &stop, const std::string &text) {+ if (!text.empty() && !stop.empty()) {+ const char text_last_char = text.back();+ for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) {+ if (stop[char_index] == text_last_char) {+ const std::string current_partial = stop.substr(0, char_index + 1);+ if (ends_with(text, current_partial)) {+ return text.size() - char_index - 1;+ }+ }+ }+ }++ return std::string::npos;+}++// TODO: reuse llama_detokenize+template <class Iter>+static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) {+ std::string ret;+ for (; begin != end; ++begin) {+ ret += llama_token_to_piece(ctx, *begin);+ }++ return ret;+}++// format incomplete utf-8 multibyte character for output+static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {+ std::string out = token == -1 ? "" : llama_token_to_piece(ctx, token);++ // if the size is 1 and first bit is 1, meaning it's a partial character+ // (size > 1 meaning it's already a known token)+ if (out.size() == 1 && (out[0] & 0x80) == 0x80) {+ std::stringstream ss;+ ss << std::hex << (out[0] & 0xff);+ std::string res(ss.str());+ out = "byte: \\x" + res;+ }++ return out;+}++struct completion_token_output {+ llama_token tok;+ std::string text_to_send;++ struct token_prob {+ llama_token tok;+ float prob;+ };++ std::vector<token_prob> probs;+};++// convert a vector of completion_token_output to json+static json probs_vector_to_json(const llama_context * ctx, const std::vector<completion_token_output> & probs) {+ json out = json::array();++ for (const auto & prob : probs) {+ json probs_for_token = json::array();++ for (const auto & p : prob.probs) {+ const std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);+ probs_for_token.push_back(json {+ {"tok_str", tok_str},+ {"prob", p.prob},+ });+ }++ const std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);+ out.push_back(json {+ {"content", tok_str},+ {"probs", probs_for_token},+ });+ }++ return out;+}++//+// OAI utils+//++static json oaicompat_completion_params_parse(+ const struct llama_model * model,+ const json & body, /* openai api json semantics */+ const std::string & chat_template) {+ json llama_params;++ llama_params["__oaicompat"] = true;++ // Map OpenAI parameters to llama.cpp parameters+ //+ // For parameters that are defined by the OpenAI documentation (e.g.+ // temperature), we explicitly specify OpenAI's intended default; we+ // need to do that because sometimes OpenAI disagrees with llama.cpp+ //+ // https://platform.openai.com/docs/api-reference/chat/create+ llama_sampling_params default_sparams;+ llama_params["model"] = json_value(body, "model", std::string("unknown"));+ llama_params["prompt"] = format_chat(model, chat_template, body["messages"]);+ llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);+ llama_params["temperature"] = json_value(body, "temperature", 0.0);+ llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);+ llama_params["top_p"] = json_value(body, "top_p", 1.0);+ llama_params["n_predict"] = json_value(body, "max_tokens", -1);+ llama_params["logit_bias"] = json_value(body, "logit_bias", json::object());+ llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);+ llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);+ llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);+ llama_params["stream"] = json_value(body, "stream", false);+ llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);+ llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);+ llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);+ llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);+ llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);+ llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);+ llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);+ llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);+ llama_params["n_keep"] = json_value(body, "n_keep", 0);++ if (body.contains("grammar")) {+ llama_params["grammar"] = json_value(body, "grammar", json::object());+ }++ if (body.contains("response_format")) {+ auto response_format = json_value(body, "response_format", json::object());+ if (response_format.contains("type")) {+ if (response_format["type"] == "json_object") {+ llama_params["json_schema"] = json_value(response_format, "schema", json::object());+ } else {+ throw std::runtime_error("response_format type not supported: " + response_format["type"].dump());+ }+ }+ }++ // Handle 'stop' field+ if (body.contains("stop") && body["stop"].is_string()) {+ llama_params["stop"] = json::array({body["stop"].get<std::string>()});+ } else {+ llama_params["stop"] = json_value(body, "stop", json::array());+ }++ // Ensure there is ChatML-specific end sequence among stop words+ llama_params["stop"].push_back("<|im_end|>");++ return llama_params;+}++static json format_final_response_oaicompat(const json & request, json result, const std::string & completion_id, bool streaming = false) {+ bool stopped_word = result.count("stopped_word") != 0;+ bool stopped_eos = json_value(result, "stopped_eos", false);+ int num_tokens_predicted = json_value(result, "tokens_predicted", 0);+ int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);+ std::string content = json_value(result, "content", std::string(""));++ std::string finish_reason = "length";+ if (stopped_word || stopped_eos) {+ finish_reason = "stop";+ }++ json choices =+ streaming ? json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"delta", json::object()}}})+ : json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"message", json{{"content", content},+ {"role", "assistant"}}}}});++ std::time_t t = std::time(0);++ json res = json {+ {"choices", choices},+ {"created", t},+ {"model",+ json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},+ {"object", streaming ? "chat.completion.chunk" : "chat.completion"},+ {"usage", json {+ {"completion_tokens", num_tokens_predicted},+ {"prompt_tokens", num_prompt_tokens},+ {"total_tokens", num_tokens_predicted + num_prompt_tokens}+ }},+ {"id", completion_id}+ };++ if (server_verbose) {+ res["__verbose"] = result;+ }++ if (result.contains("completion_probabilities")) {+ res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());+ }++ return res;+}++// return value is vector as there is one case where we might need to generate two responses+static std::vector<json> format_partial_response_oaicompat(json result, const std::string & completion_id) {+ if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {+ return std::vector<json>({result});+ }++ bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;+ std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));++ bool stopped_word = json_value(result, "stopped_word", false);+ bool stopped_eos = json_value(result, "stopped_eos", false);+ bool stopped_limit = json_value(result, "stopped_limit", false);+ std::string content = json_value(result, "content", std::string(""));++ std::string finish_reason;+ if (stopped_word || stopped_eos) {+ finish_reason = "stop";+ }+ if (stopped_limit) {+ finish_reason = "length";+ }++ std::time_t t = std::time(0);++ json choices;++ if (!finish_reason.empty()) {+ choices = json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"delta", json::object()}}});+ } else {+ if (first) {+ if (content.empty()) {+ choices = json::array({json{{"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{{"role", "assistant"}}}}});+ } else {+ // We have to send this as two updates to conform to openai behavior+ json initial_ret = json{{"choices", json::array({json{+ {"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{+ {"role", "assistant"}+ }}}})},+ {"created", t},+ {"id", completion_id},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ json second_ret = json{+ {"choices", json::array({json{{"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{+ {"content", content}}}+ }})},+ {"created", t},+ {"id", completion_id},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ return std::vector<json>({initial_ret, second_ret});+ }+ } else {+ // Some idiosyncrasy in task processing logic makes several trailing calls+ // with empty content, we ignore these at the calee site.+ if (content.empty()) {+ return std::vector<json>({json::object()});+ }++ choices = json::array({json{+ {"finish_reason", nullptr},+ {"index", 0},+ {"delta",+ json{+ {"content", content},+ }},+ }});+ }+ }++ json ret = json {+ {"choices", choices},+ {"created", t},+ {"id", completion_id},+ {"model", modelname},+ {"object", "chat.completion.chunk"}+ };++ return std::vector<json>({ret});+}++static json format_embeddings_response_oaicompat(const json & request, const json & embeddings) {+ json data = json::array();+ int i = 0;+ for (auto & elem : embeddings) {+ data.push_back(json{+ {"embedding", json_value(elem, "embedding", json::array())},+ {"index", i++},+ {"object", "embedding"}+ });+ }++ json res = json {+ {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},+ {"object", "list"},+ {"usage", json {+ {"prompt_tokens", 0},+ {"total_tokens", 0}+ }},+ {"data", data}+ };++ return res;+}++static json format_tokenizer_response(const std::vector<llama_token> & tokens) {+ return json {+ {"tokens", tokens}+ };+}++static json format_detokenized_response(const std::string & content) {+ return json {+ {"content", content}+ };+}++static json format_error_response(const std::string & message, const enum error_type type) {+ std::string type_str;+ int code = 500;+ switch (type) {+ case ERROR_TYPE_INVALID_REQUEST:+ type_str = "invalid_request_error";+ code = 400;+ break;+ case ERROR_TYPE_AUTHENTICATION:+ type_str = "authentication_error";+ code = 401;+ break;+ case ERROR_TYPE_NOT_FOUND:+ type_str = "not_found_error";+ code = 404;+ break;+ case ERROR_TYPE_SERVER:+ type_str = "server_error";+ code = 500;+ break;+ case ERROR_TYPE_PERMISSION:+ type_str = "permission_error";+ code = 403;+ break;+ case ERROR_TYPE_NOT_SUPPORTED:+ type_str = "not_supported_error";+ code = 501;+ break;+ case ERROR_TYPE_UNAVAILABLE:+ type_str = "unavailable_error";+ code = 503;+ break;+ }+ return json {+ {"code", code},+ {"message", message},+ {"type", type_str},+ };+}diff --git a/llama.cpp/unicode.cpp b/llama.cpp/unicode.cppnew file mode 100644index 00000000..7fce6fb3--- /dev/null+++ b/llama.cpp/unicode.cpp@@ -0,0 +1,1672 @@+#include "unicode.h"++#include <cassert>+#include <map>+#include <stdexcept>+#include <string>+#include <unordered_map>+#include <vector>++static const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_digit = {+{0x00000030, 0x00000039}, {0x000000B2, 0x000000B3}, {0x000000B9, 0x000000B9}, {0x00000660, 0x00000669},+{0x000006F0, 0x000006F9}, {0x000007C0, 0x000007C9}, {0x00000966, 0x0000096F}, {0x000009E6, 0x000009EF},+{0x00000A66, 0x00000A6F}, {0x00000AE6, 0x00000AEF}, {0x00000B66, 0x00000B6F}, {0x00000BE6, 0x00000BEF},+{0x00000C66, 0x00000C6F}, {0x00000CE6, 0x00000CEF}, {0x00000D66, 0x00000D6F}, {0x00000DE6, 0x00000DEF},+{0x00000E50, 0x00000E59}, {0x00000ED0, 0x00000ED9}, {0x00000F20, 0x00000F29}, {0x00001040, 0x00001049},+{0x00001090, 0x00001099}, {0x00001369, 0x00001371}, {0x000017E0, 0x000017E9}, {0x00001810, 0x00001819},+{0x00001946, 0x0000194F}, {0x000019D0, 0x000019DA}, {0x00001A80, 0x00001A89}, {0x00001A90, 0x00001A99},+{0x00001B50, 0x00001B59}, {0x00001BB0, 0x00001BB9}, {0x00001C40, 0x00001C49}, {0x00001C50, 0x00001C59},+{0x00002070, 0x00002070}, {0x00002074, 0x00002079}, {0x00002080, 0x00002089}, {0x00002460, 0x00002468},+{0x00002474, 0x0000247C}, {0x00002488, 0x00002490}, {0x000024EA, 0x000024EA}, {0x000024F5, 0x000024FD},+{0x000024FF, 0x000024FF}, {0x00002776, 0x0000277E}, {0x00002780, 0x00002788}, {0x0000278A, 0x00002792},+{0x0000A620, 0x0000A629}, {0x0000A8D0, 0x0000A8D9}, {0x0000A900, 0x0000A909}, {0x0000A9D0, 0x0000A9D9},+{0x0000A9F0, 0x0000A9F9}, {0x0000AA50, 0x0000AA59}, {0x0000ABF0, 0x0000ABF9}, {0x0000FF10, 0x0000FF19},+{0x000104A0, 0x000104A9}, {0x00010A40, 0x00010A43}, {0x00010D30, 0x00010D39}, {0x00010E60, 0x00010E68},+{0x00011052, 0x0001105A}, {0x00011066, 0x0001106F}, {0x000110F0, 0x000110F9}, {0x00011136, 0x0001113F},+{0x000111D0, 0x000111D9}, {0x000112F0, 0x000112F9}, {0x00011450, 0x00011459}, {0x000114D0, 0x000114D9},+{0x00011650, 0x00011659}, {0x000116C0, 0x000116C9}, {0x00011730, 0x00011739}, {0x000118E0, 0x000118E9},+{0x00011950, 0x00011959}, {0x00011C50, 0x00011C59}, {0x00011D50, 0x00011D59}, {0x00011DA0, 0x00011DA9},+{0x00016A60, 0x00016A69}, {0x00016B50, 0x00016B59}, {0x0001D7CE, 0x0001D7FF}, {0x0001E140, 0x0001E149},+{0x0001E2F0, 0x0001E2F9}, {0x0001E950, 0x0001E959}, {0x0001F100, 0x0001F10A}, {0x0001FBF0, 0x0001FBF9},+};++static const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_letter = {+{0x00000041, 0x0000005A}, {0x00000061, 0x0000007A}, {0x000000AA, 0x000000AA}, {0x000000B5, 0x000000B5},+{0x000000BA, 0x000000BA}, {0x000000C0, 0x000000D6}, {0x000000D8, 0x000000F6}, {0x000000F8, 0x000002C1},+{0x000002C6, 0x000002D1}, {0x000002E0, 0x000002E4}, {0x000002EC, 0x000002EC}, {0x000002EE, 0x000002EE},+{0x00000370, 0x00000374}, {0x00000376, 0x00000377}, {0x0000037A, 0x0000037D}, {0x0000037F, 0x0000037F},+{0x00000386, 0x00000386}, {0x00000388, 0x0000038A}, {0x0000038C, 0x0000038C}, {0x0000038E, 0x000003A1},+{0x000003A3, 0x000003F5}, {0x000003F7, 0x00000481}, {0x0000048A, 0x0000052F}, {0x00000531, 0x00000556},+{0x00000559, 0x00000559}, {0x00000560, 0x00000588}, {0x000005D0, 0x000005EA}, {0x000005EF, 0x000005F2},+{0x00000620, 0x0000064A}, {0x0000066E, 0x0000066F}, {0x00000671, 0x000006D3}, {0x000006D5, 0x000006D5},+{0x000006E5, 0x000006E6}, {0x000006EE, 0x000006EF}, {0x000006FA, 0x000006FC}, {0x000006FF, 0x000006FF},+{0x00000710, 0x00000710}, {0x00000712, 0x0000072F}, {0x0000074D, 0x000007A5}, {0x000007B1, 0x000007B1},+{0x000007CA, 0x000007EA}, {0x000007F4, 0x000007F5}, {0x000007FA, 0x000007FA}, {0x00000800, 0x00000815},+{0x0000081A, 0x0000081A}, {0x00000824, 0x00000824}, {0x00000828, 0x00000828}, {0x00000840, 0x00000858},+{0x00000860, 0x0000086A}, {0x000008A0, 0x000008B4}, {0x000008B6, 0x000008C7}, {0x00000904, 0x00000939},+{0x0000093D, 0x0000093D}, {0x00000950, 0x00000950}, {0x00000958, 0x00000961}, {0x00000971, 0x00000980},+{0x00000985, 0x0000098C}, {0x0000098F, 0x00000990}, {0x00000993, 0x000009A8}, {0x000009AA, 0x000009B0},+{0x000009B2, 0x000009B2}, {0x000009B6, 0x000009B9}, {0x000009BD, 0x000009BD}, {0x000009CE, 0x000009CE},+{0x000009DC, 0x000009DD}, {0x000009DF, 0x000009E1}, {0x000009F0, 0x000009F1}, {0x000009FC, 0x000009FC},+{0x00000A05, 0x00000A0A}, {0x00000A0F, 0x00000A10}, {0x00000A13, 0x00000A28}, {0x00000A2A, 0x00000A30},+{0x00000A32, 0x00000A33}, {0x00000A35, 0x00000A36}, {0x00000A38, 0x00000A39}, {0x00000A59, 0x00000A5C},+{0x00000A5E, 0x00000A5E}, {0x00000A72, 0x00000A74}, {0x00000A85, 0x00000A8D}, {0x00000A8F, 0x00000A91},+{0x00000A93, 0x00000AA8}, {0x00000AAA, 0x00000AB0}, {0x00000AB2, 0x00000AB3}, {0x00000AB5, 0x00000AB9},+{0x00000ABD, 0x00000ABD}, {0x00000AD0, 0x00000AD0}, {0x00000AE0, 0x00000AE1}, {0x00000AF9, 0x00000AF9},+{0x00000B05, 0x00000B0C}, {0x00000B0F, 0x00000B10}, {0x00000B13, 0x00000B28}, {0x00000B2A, 0x00000B30},+{0x00000B32, 0x00000B33}, {0x00000B35, 0x00000B39}, {0x00000B3D, 0x00000B3D}, {0x00000B5C, 0x00000B5D},+{0x00000B5F, 0x00000B61}, {0x00000B71, 0x00000B71}, {0x00000B83, 0x00000B83}, {0x00000B85, 0x00000B8A},+{0x00000B8E, 0x00000B90}, {0x00000B92, 0x00000B95}, {0x00000B99, 0x00000B9A}, {0x00000B9C, 0x00000B9C},+{0x00000B9E, 0x00000B9F}, {0x00000BA3, 0x00000BA4}, {0x00000BA8, 0x00000BAA}, {0x00000BAE, 0x00000BB9},+{0x00000BD0, 0x00000BD0}, {0x00000C05, 0x00000C0C}, {0x00000C0E, 0x00000C10}, {0x00000C12, 0x00000C28},+{0x00000C2A, 0x00000C39}, {0x00000C3D, 0x00000C3D}, {0x00000C58, 0x00000C5A}, {0x00000C60, 0x00000C61},+{0x00000C80, 0x00000C80}, {0x00000C85, 0x00000C8C}, {0x00000C8E, 0x00000C90}, {0x00000C92, 0x00000CA8},+{0x00000CAA, 0x00000CB3}, {0x00000CB5, 0x00000CB9}, {0x00000CBD, 0x00000CBD}, {0x00000CDE, 0x00000CDE},+{0x00000CE0, 0x00000CE1}, {0x00000CF1, 0x00000CF2}, {0x00000D04, 0x00000D0C}, {0x00000D0E, 0x00000D10},+{0x00000D12, 0x00000D3A}, {0x00000D3D, 0x00000D3D}, {0x00000D4E, 0x00000D4E}, {0x00000D54, 0x00000D56},+{0x00000D5F, 0x00000D61}, {0x00000D7A, 0x00000D7F}, {0x00000D85, 0x00000D96}, {0x00000D9A, 0x00000DB1},+{0x00000DB3, 0x00000DBB}, {0x00000DBD, 0x00000DBD}, {0x00000DC0, 0x00000DC6}, {0x00000E01, 0x00000E30},+{0x00000E32, 0x00000E33}, {0x00000E40, 0x00000E46}, {0x00000E81, 0x00000E82}, {0x00000E84, 0x00000E84},+{0x00000E86, 0x00000E8A}, {0x00000E8C, 0x00000EA3}, {0x00000EA5, 0x00000EA5}, {0x00000EA7, 0x00000EB0},+{0x00000EB2, 0x00000EB3}, {0x00000EBD, 0x00000EBD}, {0x00000EC0, 0x00000EC4}, {0x00000EC6, 0x00000EC6},+{0x00000EDC, 0x00000EDF}, {0x00000F00, 0x00000F00}, {0x00000F40, 0x00000F47}, {0x00000F49, 0x00000F6C},+{0x00000F88, 0x00000F8C}, {0x00001000, 0x0000102A}, {0x0000103F, 0x0000103F}, {0x00001050, 0x00001055},+{0x0000105A, 0x0000105D}, {0x00001061, 0x00001061}, {0x00001065, 0x00001066}, {0x0000106E, 0x00001070},+{0x00001075, 0x00001081}, {0x0000108E, 0x0000108E}, {0x000010A0, 0x000010C5}, {0x000010C7, 0x000010C7},+{0x000010CD, 0x000010CD}, {0x000010D0, 0x000010FA}, {0x000010FC, 0x00001248}, {0x0000124A, 0x0000124D},+{0x00001250, 0x00001256}, {0x00001258, 0x00001258}, {0x0000125A, 0x0000125D}, {0x00001260, 0x00001288},+{0x0000128A, 0x0000128D}, {0x00001290, 0x000012B0}, {0x000012B2, 0x000012B5}, {0x000012B8, 0x000012BE},+{0x000012C0, 0x000012C0}, {0x000012C2, 0x000012C5}, {0x000012C8, 0x000012D6}, {0x000012D8, 0x00001310},+{0x00001312, 0x00001315}, {0x00001318, 0x0000135A}, {0x00001380, 0x0000138F}, {0x000013A0, 0x000013F5},+{0x000013F8, 0x000013FD}, {0x00001401, 0x0000166C}, {0x0000166F, 0x0000167F}, {0x00001681, 0x0000169A},+{0x000016A0, 0x000016EA}, {0x000016F1, 0x000016F8}, {0x00001700, 0x0000170C}, {0x0000170E, 0x00001711},+{0x00001720, 0x00001731}, {0x00001740, 0x00001751}, {0x00001760, 0x0000176C}, {0x0000176E, 0x00001770},+{0x00001780, 0x000017B3}, {0x000017D7, 0x000017D7}, {0x000017DC, 0x000017DC}, {0x00001820, 0x00001878},+{0x00001880, 0x00001884}, {0x00001887, 0x000018A8}, {0x000018AA, 0x000018AA}, {0x000018B0, 0x000018F5},+{0x00001900, 0x0000191E}, {0x00001950, 0x0000196D}, {0x00001970, 0x00001974}, {0x00001980, 0x000019AB},+{0x000019B0, 0x000019C9}, {0x00001A00, 0x00001A16}, {0x00001A20, 0x00001A54}, {0x00001AA7, 0x00001AA7},+{0x00001B05, 0x00001B33}, {0x00001B45, 0x00001B4B}, {0x00001B83, 0x00001BA0}, {0x00001BAE, 0x00001BAF},+{0x00001BBA, 0x00001BE5}, {0x00001C00, 0x00001C23}, {0x00001C4D, 0x00001C4F}, {0x00001C5A, 0x00001C7D},+{0x00001C80, 0x00001C88}, {0x00001C90, 0x00001CBA}, {0x00001CBD, 0x00001CBF}, {0x00001CE9, 0x00001CEC},+{0x00001CEE, 0x00001CF3}, {0x00001CF5, 0x00001CF6}, {0x00001CFA, 0x00001CFA}, {0x00001D00, 0x00001DBF},+{0x00001E00, 0x00001F15}, {0x00001F18, 0x00001F1D}, {0x00001F20, 0x00001F45}, {0x00001F48, 0x00001F4D},+{0x00001F50, 0x00001F57}, {0x00001F59, 0x00001F59}, {0x00001F5B, 0x00001F5B}, {0x00001F5D, 0x00001F5D},+{0x00001F5F, 0x00001F7D}, {0x00001F80, 0x00001FB4}, {0x00001FB6, 0x00001FBC}, {0x00001FBE, 0x00001FBE},+{0x00001FC2, 0x00001FC4}, {0x00001FC6, 0x00001FCC}, {0x00001FD0, 0x00001FD3}, {0x00001FD6, 0x00001FDB},+{0x00001FE0, 0x00001FEC}, {0x00001FF2, 0x00001FF4}, {0x00001FF6, 0x00001FFC}, {0x00002071, 0x00002071},+{0x0000207F, 0x0000207F}, {0x00002090, 0x0000209C}, {0x00002102, 0x00002102}, {0x00002107, 0x00002107},+{0x0000210A, 0x00002113}, {0x00002115, 0x00002115}, {0x00002119, 0x0000211D}, {0x00002124, 0x00002124},+{0x00002126, 0x00002126}, {0x00002128, 0x00002128}, {0x0000212A, 0x0000212D}, {0x0000212F, 0x00002139},+{0x0000213C, 0x0000213F}, {0x00002145, 0x00002149}, {0x0000214E, 0x0000214E}, {0x00002183, 0x00002184},+{0x00002C00, 0x00002C2E}, {0x00002C30, 0x00002C5E}, {0x00002C60, 0x00002CE4}, {0x00002CEB, 0x00002CEE},+{0x00002CF2, 0x00002CF3}, {0x00002D00, 0x00002D25}, {0x00002D27, 0x00002D27}, {0x00002D2D, 0x00002D2D},+{0x00002D30, 0x00002D67}, {0x00002D6F, 0x00002D6F}, {0x00002D80, 0x00002D96}, {0x00002DA0, 0x00002DA6},+{0x00002DA8, 0x00002DAE}, {0x00002DB0, 0x00002DB6}, {0x00002DB8, 0x00002DBE}, {0x00002DC0, 0x00002DC6},+{0x00002DC8, 0x00002DCE}, {0x00002DD0, 0x00002DD6}, {0x00002DD8, 0x00002DDE}, {0x00002E2F, 0x00002E2F},+{0x00003005, 0x00003006}, {0x00003031, 0x00003035}, {0x0000303B, 0x0000303C}, {0x00003041, 0x00003096},+{0x0000309D, 0x0000309F}, {0x000030A1, 0x000030FA}, {0x000030FC, 0x000030FF}, {0x00003105, 0x0000312F},+{0x00003131, 0x0000318E}, {0x000031A0, 0x000031BF}, {0x000031F0, 0x000031FF}, {0x00003400, 0x00004DBF},+{0x00004E00, 0x00009FFC}, {0x0000A000, 0x0000A48C}, {0x0000A4D0, 0x0000A4FD}, {0x0000A500, 0x0000A60C},+{0x0000A610, 0x0000A61F}, {0x0000A62A, 0x0000A62B}, {0x0000A640, 0x0000A66E}, {0x0000A67F, 0x0000A69D},+{0x0000A6A0, 0x0000A6E5}, {0x0000A717, 0x0000A71F}, {0x0000A722, 0x0000A788}, {0x0000A78B, 0x0000A7BF},+{0x0000A7C2, 0x0000A7CA}, {0x0000A7F5, 0x0000A801}, {0x0000A803, 0x0000A805}, {0x0000A807, 0x0000A80A},+{0x0000A80C, 0x0000A822}, {0x0000A840, 0x0000A873}, {0x0000A882, 0x0000A8B3}, {0x0000A8F2, 0x0000A8F7},+{0x0000A8FB, 0x0000A8FB}, {0x0000A8FD, 0x0000A8FE}, {0x0000A90A, 0x0000A925}, {0x0000A930, 0x0000A946},+{0x0000A960, 0x0000A97C}, {0x0000A984, 0x0000A9B2}, {0x0000A9CF, 0x0000A9CF}, {0x0000A9E0, 0x0000A9E4},+{0x0000A9E6, 0x0000A9EF}, {0x0000A9FA, 0x0000A9FE}, {0x0000AA00, 0x0000AA28}, {0x0000AA40, 0x0000AA42},+{0x0000AA44, 0x0000AA4B}, {0x0000AA60, 0x0000AA76}, {0x0000AA7A, 0x0000AA7A}, {0x0000AA7E, 0x0000AAAF},+{0x0000AAB1, 0x0000AAB1}, {0x0000AAB5, 0x0000AAB6}, {0x0000AAB9, 0x0000AABD}, {0x0000AAC0, 0x0000AAC0},+{0x0000AAC2, 0x0000AAC2}, {0x0000AADB, 0x0000AADD}, {0x0000AAE0, 0x0000AAEA}, {0x0000AAF2, 0x0000AAF4},+{0x0000AB01, 0x0000AB06}, {0x0000AB09, 0x0000AB0E}, {0x0000AB11, 0x0000AB16}, {0x0000AB20, 0x0000AB26},+{0x0000AB28, 0x0000AB2E}, {0x0000AB30, 0x0000AB5A}, {0x0000AB5C, 0x0000AB69}, {0x0000AB70, 0x0000ABE2},+{0x0000AC00, 0x0000D7A3}, {0x0000D7B0, 0x0000D7C6}, {0x0000D7CB, 0x0000D7FB}, {0x0000F900, 0x0000FA6D},+{0x0000FA70, 0x0000FAD9}, {0x0000FB00, 0x0000FB06}, {0x0000FB13, 0x0000FB17}, {0x0000FB1D, 0x0000FB1D},+{0x0000FB1F, 0x0000FB28}, {0x0000FB2A, 0x0000FB36}, {0x0000FB38, 0x0000FB3C}, {0x0000FB3E, 0x0000FB3E},+{0x0000FB40, 0x0000FB41}, {0x0000FB43, 0x0000FB44}, {0x0000FB46, 0x0000FBB1}, {0x0000FBD3, 0x0000FD3D},+{0x0000FD50, 0x0000FD8F}, {0x0000FD92, 0x0000FDC7}, {0x0000FDF0, 0x0000FDFB}, {0x0000FE70, 0x0000FE74},+{0x0000FE76, 0x0000FEFC}, {0x0000FF21, 0x0000FF3A}, {0x0000FF41, 0x0000FF5A}, {0x0000FF66, 0x0000FFBE},+{0x0000FFC2, 0x0000FFC7}, {0x0000FFCA, 0x0000FFCF}, {0x0000FFD2, 0x0000FFD7}, {0x0000FFDA, 0x0000FFDC},+{0x00010000, 0x0001000B}, {0x0001000D, 0x00010026}, {0x00010028, 0x0001003A}, {0x0001003C, 0x0001003D},+{0x0001003F, 0x0001004D}, {0x00010050, 0x0001005D}, {0x00010080, 0x000100FA}, {0x00010280, 0x0001029C},+{0x000102A0, 0x000102D0}, {0x00010300, 0x0001031F}, {0x0001032D, 0x00010340}, {0x00010342, 0x00010349},+{0x00010350, 0x00010375}, {0x00010380, 0x0001039D}, {0x000103A0, 0x000103C3}, {0x000103C8, 0x000103CF},+{0x00010400, 0x0001049D}, {0x000104B0, 0x000104D3}, {0x000104D8, 0x000104FB}, {0x00010500, 0x00010527},+{0x00010530, 0x00010563}, {0x00010600, 0x00010736}, {0x00010740, 0x00010755}, {0x00010760, 0x00010767},+{0x00010800, 0x00010805}, {0x00010808, 0x00010808}, {0x0001080A, 0x00010835}, {0x00010837, 0x00010838},+{0x0001083C, 0x0001083C}, {0x0001083F, 0x00010855}, {0x00010860, 0x00010876}, {0x00010880, 0x0001089E},+{0x000108E0, 0x000108F2}, {0x000108F4, 0x000108F5}, {0x00010900, 0x00010915}, {0x00010920, 0x00010939},+{0x00010980, 0x000109B7}, {0x000109BE, 0x000109BF}, {0x00010A00, 0x00010A00}, {0x00010A10, 0x00010A13},+{0x00010A15, 0x00010A17}, {0x00010A19, 0x00010A35}, {0x00010A60, 0x00010A7C}, {0x00010A80, 0x00010A9C},+{0x00010AC0, 0x00010AC7}, {0x00010AC9, 0x00010AE4}, {0x00010B00, 0x00010B35}, {0x00010B40, 0x00010B55},+{0x00010B60, 0x00010B72}, {0x00010B80, 0x00010B91}, {0x00010C00, 0x00010C48}, {0x00010C80, 0x00010CB2},+{0x00010CC0, 0x00010CF2}, {0x00010D00, 0x00010D23}, {0x00010E80, 0x00010EA9}, {0x00010EB0, 0x00010EB1},+{0x00010F00, 0x00010F1C}, {0x00010F27, 0x00010F27}, {0x00010F30, 0x00010F45}, {0x00010FB0, 0x00010FC4},+{0x00010FE0, 0x00010FF6}, {0x00011003, 0x00011037}, {0x00011083, 0x000110AF}, {0x000110D0, 0x000110E8},+{0x00011103, 0x00011126}, {0x00011144, 0x00011144}, {0x00011147, 0x00011147}, {0x00011150, 0x00011172},+{0x00011176, 0x00011176}, {0x00011183, 0x000111B2}, {0x000111C1, 0x000111C4}, {0x000111DA, 0x000111DA},+{0x000111DC, 0x000111DC}, {0x00011200, 0x00011211}, {0x00011213, 0x0001122B}, {0x00011280, 0x00011286},+{0x00011288, 0x00011288}, {0x0001128A, 0x0001128D}, {0x0001128F, 0x0001129D}, {0x0001129F, 0x000112A8},+{0x000112B0, 0x000112DE}, {0x00011305, 0x0001130C}, {0x0001130F, 0x00011310}, {0x00011313, 0x00011328},+{0x0001132A, 0x00011330}, {0x00011332, 0x00011333}, {0x00011335, 0x00011339}, {0x0001133D, 0x0001133D},+{0x00011350, 0x00011350}, {0x0001135D, 0x00011361}, {0x00011400, 0x00011434}, {0x00011447, 0x0001144A},+{0x0001145F, 0x00011461}, {0x00011480, 0x000114AF}, {0x000114C4, 0x000114C5}, {0x000114C7, 0x000114C7},+{0x00011580, 0x000115AE}, {0x000115D8, 0x000115DB}, {0x00011600, 0x0001162F}, {0x00011644, 0x00011644},+{0x00011680, 0x000116AA}, {0x000116B8, 0x000116B8}, {0x00011700, 0x0001171A}, {0x00011800, 0x0001182B},+{0x000118A0, 0x000118DF}, {0x000118FF, 0x00011906}, {0x00011909, 0x00011909}, {0x0001190C, 0x00011913},+{0x00011915, 0x00011916}, {0x00011918, 0x0001192F}, {0x0001193F, 0x0001193F}, {0x00011941, 0x00011941},+{0x000119A0, 0x000119A7}, {0x000119AA, 0x000119D0}, {0x000119E1, 0x000119E1}, {0x000119E3, 0x000119E3},+{0x00011A00, 0x00011A00}, {0x00011A0B, 0x00011A32}, {0x00011A3A, 0x00011A3A}, {0x00011A50, 0x00011A50},+{0x00011A5C, 0x00011A89}, {0x00011A9D, 0x00011A9D}, {0x00011AC0, 0x00011AF8}, {0x00011C00, 0x00011C08},+{0x00011C0A, 0x00011C2E}, {0x00011C40, 0x00011C40}, {0x00011C72, 0x00011C8F}, {0x00011D00, 0x00011D06},+{0x00011D08, 0x00011D09}, {0x00011D0B, 0x00011D30}, {0x00011D46, 0x00011D46}, {0x00011D60, 0x00011D65},+{0x00011D67, 0x00011D68}, {0x00011D6A, 0x00011D89}, {0x00011D98, 0x00011D98}, {0x00011EE0, 0x00011EF2},+{0x00011FB0, 0x00011FB0}, {0x00012000, 0x00012399}, {0x00012480, 0x00012543}, {0x00013000, 0x0001342E},+{0x00014400, 0x00014646}, {0x00016800, 0x00016A38}, {0x00016A40, 0x00016A5E}, {0x00016AD0, 0x00016AED},+{0x00016B00, 0x00016B2F}, {0x00016B40, 0x00016B43}, {0x00016B63, 0x00016B77}, {0x00016B7D, 0x00016B8F},+{0x00016E40, 0x00016E7F}, {0x00016F00, 0x00016F4A}, {0x00016F50, 0x00016F50}, {0x00016F93, 0x00016F9F},+{0x00016FE0, 0x00016FE1}, {0x00016FE3, 0x00016FE3}, {0x00017000, 0x000187F7}, {0x00018800, 0x00018CD5},+{0x00018D00, 0x00018D08}, {0x0001B000, 0x0001B11E}, {0x0001B150, 0x0001B152}, {0x0001B164, 0x0001B167},+{0x0001B170, 0x0001B2FB}, {0x0001BC00, 0x0001BC6A}, {0x0001BC70, 0x0001BC7C}, {0x0001BC80, 0x0001BC88},+{0x0001BC90, 0x0001BC99}, {0x0001D400, 0x0001D454}, {0x0001D456, 0x0001D49C}, {0x0001D49E, 0x0001D49F},+{0x0001D4A2, 0x0001D4A2}, {0x0001D4A5, 0x0001D4A6}, {0x0001D4A9, 0x0001D4AC}, {0x0001D4AE, 0x0001D4B9},+{0x0001D4BB, 0x0001D4BB}, {0x0001D4BD, 0x0001D4C3}, {0x0001D4C5, 0x0001D505}, {0x0001D507, 0x0001D50A},+{0x0001D50D, 0x0001D514}, {0x0001D516, 0x0001D51C}, {0x0001D51E, 0x0001D539}, {0x0001D53B, 0x0001D53E},+{0x0001D540, 0x0001D544}, {0x0001D546, 0x0001D546}, {0x0001D54A, 0x0001D550}, {0x0001D552, 0x0001D6A5},+{0x0001D6A8, 0x0001D6C0}, {0x0001D6C2, 0x0001D6DA}, {0x0001D6DC, 0x0001D6FA}, {0x0001D6FC, 0x0001D714},+{0x0001D716, 0x0001D734}, {0x0001D736, 0x0001D74E}, {0x0001D750, 0x0001D76E}, {0x0001D770, 0x0001D788},+{0x0001D78A, 0x0001D7A8}, {0x0001D7AA, 0x0001D7C2}, {0x0001D7C4, 0x0001D7CB}, {0x0001E100, 0x0001E12C},+{0x0001E137, 0x0001E13D}, {0x0001E14E, 0x0001E14E}, {0x0001E2C0, 0x0001E2EB}, {0x0001E800, 0x0001E8C4},+{0x0001E900, 0x0001E943}, {0x0001E94B, 0x0001E94B}, {0x0001EE00, 0x0001EE03}, {0x0001EE05, 0x0001EE1F},+{0x0001EE21, 0x0001EE22}, {0x0001EE24, 0x0001EE24}, {0x0001EE27, 0x0001EE27}, {0x0001EE29, 0x0001EE32},+{0x0001EE34, 0x0001EE37}, {0x0001EE39, 0x0001EE39}, {0x0001EE3B, 0x0001EE3B}, {0x0001EE42, 0x0001EE42},+{0x0001EE47, 0x0001EE47}, {0x0001EE49, 0x0001EE49}, {0x0001EE4B, 0x0001EE4B}, {0x0001EE4D, 0x0001EE4F},+{0x0001EE51, 0x0001EE52}, {0x0001EE54, 0x0001EE54}, {0x0001EE57, 0x0001EE57}, {0x0001EE59, 0x0001EE59},+{0x0001EE5B, 0x0001EE5B}, {0x0001EE5D, 0x0001EE5D}, {0x0001EE5F, 0x0001EE5F}, {0x0001EE61, 0x0001EE62},+{0x0001EE64, 0x0001EE64}, {0x0001EE67, 0x0001EE6A}, {0x0001EE6C, 0x0001EE72}, {0x0001EE74, 0x0001EE77},+{0x0001EE79, 0x0001EE7C}, {0x0001EE7E, 0x0001EE7E}, {0x0001EE80, 0x0001EE89}, {0x0001EE8B, 0x0001EE9B},+{0x0001EEA1, 0x0001EEA3}, {0x0001EEA5, 0x0001EEA9}, {0x0001EEAB, 0x0001EEBB}, {0x00020000, 0x0002A6DD},+{0x0002A700, 0x0002B734}, {0x0002B740, 0x0002B81D}, {0x0002B820, 0x0002CEA1}, {0x0002CEB0, 0x0002EBE0},+{0x0002F800, 0x0002FA1D}, {0x00030000, 0x0003134A},+};++static const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_whitespace = {+{0x00000009, 0x0000000D}, {0x0000001C, 0x00000020}, {0x00000085, 0x00000085}, {0x000000A0, 0x000000A0},+{0x00001680, 0x00001680}, {0x00002000, 0x0000200A}, {0x00002028, 0x00002029}, {0x0000202F, 0x0000202F},+{0x0000205F, 0x0000205F}, {0x00003000, 0x00003000},+};++static const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_mark = {+{0x00000300, 0x0000036F}, {0x00000483, 0x00000489}, {0x00000591, 0x000005BD}, {0x000005BF, 0x000005BF},+{0x000005C1, 0x000005C2}, {0x000005C4, 0x000005C5}, {0x000005C7, 0x000005C7}, {0x00000610, 0x0000061A},+{0x0000064B, 0x0000065F}, {0x00000670, 0x00000670}, {0x000006D6, 0x000006DC}, {0x000006DF, 0x000006E4},+{0x000006E7, 0x000006E8}, {0x000006EA, 0x000006ED}, {0x00000711, 0x00000711}, {0x00000730, 0x0000074A},+{0x000007A6, 0x000007B0}, {0x000007EB, 0x000007F3}, {0x000007FD, 0x000007FD}, {0x00000816, 0x00000819},+{0x0000081B, 0x00000823}, {0x00000825, 0x00000827}, {0x00000829, 0x0000082D}, {0x00000859, 0x0000085B},+{0x000008D3, 0x000008E1}, {0x000008E3, 0x00000903}, {0x0000093A, 0x0000093C}, {0x0000093E, 0x0000094F},+{0x00000951, 0x00000957}, {0x00000962, 0x00000963}, {0x00000981, 0x00000983}, {0x000009BC, 0x000009BC},+{0x000009BE, 0x000009C4}, {0x000009C7, 0x000009C8}, {0x000009CB, 0x000009CD}, {0x000009D7, 0x000009D7},+{0x000009E2, 0x000009E3}, {0x000009FE, 0x000009FE}, {0x00000A01, 0x00000A03}, {0x00000A3C, 0x00000A3C},+{0x00000A3E, 0x00000A42}, {0x00000A47, 0x00000A48}, {0x00000A4B, 0x00000A4D}, {0x00000A51, 0x00000A51},+{0x00000A70, 0x00000A71}, {0x00000A75, 0x00000A75}, {0x00000A81, 0x00000A83}, {0x00000ABC, 0x00000ABC},+{0x00000ABE, 0x00000AC5}, {0x00000AC7, 0x00000AC9}, {0x00000ACB, 0x00000ACD}, {0x00000AE2, 0x00000AE3},+{0x00000AFA, 0x00000AFF}, {0x00000B01, 0x00000B03}, {0x00000B3C, 0x00000B3C}, {0x00000B3E, 0x00000B44},+{0x00000B47, 0x00000B48}, {0x00000B4B, 0x00000B4D}, {0x00000B55, 0x00000B57}, {0x00000B62, 0x00000B63},+{0x00000B82, 0x00000B82}, {0x00000BBE, 0x00000BC2}, {0x00000BC6, 0x00000BC8}, {0x00000BCA, 0x00000BCD},+{0x00000BD7, 0x00000BD7}, {0x00000C00, 0x00000C04}, {0x00000C3E, 0x00000C44}, {0x00000C46, 0x00000C48},+{0x00000C4A, 0x00000C4D}, {0x00000C55, 0x00000C56}, {0x00000C62, 0x00000C63}, {0x00000C81, 0x00000C83},+{0x00000CBC, 0x00000CBC}, {0x00000CBE, 0x00000CC4}, {0x00000CC6, 0x00000CC8}, {0x00000CCA, 0x00000CCD},+{0x00000CD5, 0x00000CD6}, {0x00000CE2, 0x00000CE3}, {0x00000D00, 0x00000D03}, {0x00000D3B, 0x00000D3C},+{0x00000D3E, 0x00000D44}, {0x00000D46, 0x00000D48}, {0x00000D4A, 0x00000D4D}, {0x00000D57, 0x00000D57},+{0x00000D62, 0x00000D63}, {0x00000D81, 0x00000D83}, {0x00000DCA, 0x00000DCA}, {0x00000DCF, 0x00000DD4},+{0x00000DD6, 0x00000DD6}, {0x00000DD8, 0x00000DDF}, {0x00000DF2, 0x00000DF3}, {0x00000E31, 0x00000E31},+{0x00000E34, 0x00000E3A}, {0x00000E47, 0x00000E4E}, {0x00000EB1, 0x00000EB1}, {0x00000EB4, 0x00000EBC},+{0x00000EC8, 0x00000ECD}, {0x00000F18, 0x00000F19}, {0x00000F35, 0x00000F35}, {0x00000F37, 0x00000F37},+{0x00000F39, 0x00000F39}, {0x00000F3E, 0x00000F3F}, {0x00000F71, 0x00000F84}, {0x00000F86, 0x00000F87},+{0x00000F8D, 0x00000F97}, {0x00000F99, 0x00000FBC}, {0x00000FC6, 0x00000FC6}, {0x0000102B, 0x0000103E},+{0x00001056, 0x00001059}, {0x0000105E, 0x00001060}, {0x00001062, 0x00001064}, {0x00001067, 0x0000106D},+{0x00001071, 0x00001074}, {0x00001082, 0x0000108D}, {0x0000108F, 0x0000108F}, {0x0000109A, 0x0000109D},+{0x0000135D, 0x0000135F}, {0x00001712, 0x00001714}, {0x00001732, 0x00001734}, {0x00001752, 0x00001753},+{0x00001772, 0x00001773}, {0x000017B4, 0x000017D3}, {0x000017DD, 0x000017DD}, {0x0000180B, 0x0000180D},+{0x00001885, 0x00001886}, {0x000018A9, 0x000018A9}, {0x00001920, 0x0000192B}, {0x00001930, 0x0000193B},+{0x00001A17, 0x00001A1B}, {0x00001A55, 0x00001A5E}, {0x00001A60, 0x00001A7C}, {0x00001A7F, 0x00001A7F},+{0x00001AB0, 0x00001AC0}, {0x00001B00, 0x00001B04}, {0x00001B34, 0x00001B44}, {0x00001B6B, 0x00001B73},+{0x00001B80, 0x00001B82}, {0x00001BA1, 0x00001BAD}, {0x00001BE6, 0x00001BF3}, {0x00001C24, 0x00001C37},+{0x00001CD0, 0x00001CD2}, {0x00001CD4, 0x00001CE8}, {0x00001CED, 0x00001CED}, {0x00001CF4, 0x00001CF4},+{0x00001CF7, 0x00001CF9}, {0x00001DC0, 0x00001DF9}, {0x00001DFB, 0x00001DFF}, {0x000020D0, 0x000020F0},+{0x00002CEF, 0x00002CF1}, {0x00002D7F, 0x00002D7F}, {0x00002DE0, 0x00002DFF}, {0x0000302A, 0x0000302F},+{0x00003099, 0x0000309A}, {0x0000A66F, 0x0000A672}, {0x0000A674, 0x0000A67D}, {0x0000A69E, 0x0000A69F},+{0x0000A6F0, 0x0000A6F1}, {0x0000A802, 0x0000A802}, {0x0000A806, 0x0000A806}, {0x0000A80B, 0x0000A80B},+{0x0000A823, 0x0000A827}, {0x0000A82C, 0x0000A82C}, {0x0000A880, 0x0000A881}, {0x0000A8B4, 0x0000A8C5},+{0x0000A8E0, 0x0000A8F1}, {0x0000A8FF, 0x0000A8FF}, {0x0000A926, 0x0000A92D}, {0x0000A947, 0x0000A953},+{0x0000A980, 0x0000A983}, {0x0000A9B3, 0x0000A9C0}, {0x0000A9E5, 0x0000A9E5}, {0x0000AA29, 0x0000AA36},+{0x0000AA43, 0x0000AA43}, {0x0000AA4C, 0x0000AA4D}, {0x0000AA7B, 0x0000AA7D}, {0x0000AAB0, 0x0000AAB0},+{0x0000AAB2, 0x0000AAB4}, {0x0000AAB7, 0x0000AAB8}, {0x0000AABE, 0x0000AABF}, {0x0000AAC1, 0x0000AAC1},+{0x0000AAEB, 0x0000AAEF}, {0x0000AAF5, 0x0000AAF6}, {0x0000ABE3, 0x0000ABEA}, {0x0000ABEC, 0x0000ABED},+{0x0000FB1E, 0x0000FB1E}, {0x0000FE00, 0x0000FE0F}, {0x0000FE20, 0x0000FE2F}, {0x000101FD, 0x000101FD},+{0x000102E0, 0x000102E0}, {0x00010376, 0x0001037A}, {0x00010A01, 0x00010A03}, {0x00010A05, 0x00010A06},+{0x00010A0C, 0x00010A0F}, {0x00010A38, 0x00010A3A}, {0x00010A3F, 0x00010A3F}, {0x00010AE5, 0x00010AE6},+{0x00010D24, 0x00010D27}, {0x00010EAB, 0x00010EAC}, {0x00010F46, 0x00010F50}, {0x00011000, 0x00011002},+{0x00011038, 0x00011046}, {0x0001107F, 0x00011082}, {0x000110B0, 0x000110BA}, {0x00011100, 0x00011102},+{0x00011127, 0x00011134}, {0x00011145, 0x00011146}, {0x00011173, 0x00011173}, {0x00011180, 0x00011182},+{0x000111B3, 0x000111C0}, {0x000111C9, 0x000111CC}, {0x000111CE, 0x000111CF}, {0x0001122C, 0x00011237},+{0x0001123E, 0x0001123E}, {0x000112DF, 0x000112EA}, {0x00011300, 0x00011303}, {0x0001133B, 0x0001133C},+{0x0001133E, 0x00011344}, {0x00011347, 0x00011348}, {0x0001134B, 0x0001134D}, {0x00011357, 0x00011357},+{0x00011362, 0x00011363}, {0x00011366, 0x0001136C}, {0x00011370, 0x00011374}, {0x00011435, 0x00011446},+{0x0001145E, 0x0001145E}, {0x000114B0, 0x000114C3}, {0x000115AF, 0x000115B5}, {0x000115B8, 0x000115C0},+{0x000115DC, 0x000115DD}, {0x00011630, 0x00011640}, {0x000116AB, 0x000116B7}, {0x0001171D, 0x0001172B},+{0x0001182C, 0x0001183A}, {0x00011930, 0x00011935}, {0x00011937, 0x00011938}, {0x0001193B, 0x0001193E},+{0x00011940, 0x00011940}, {0x00011942, 0x00011943}, {0x000119D1, 0x000119D7}, {0x000119DA, 0x000119E0},+{0x000119E4, 0x000119E4}, {0x00011A01, 0x00011A0A}, {0x00011A33, 0x00011A39}, {0x00011A3B, 0x00011A3E},+{0x00011A47, 0x00011A47}, {0x00011A51, 0x00011A5B}, {0x00011A8A, 0x00011A99}, {0x00011C2F, 0x00011C36},+{0x00011C38, 0x00011C3F}, {0x00011C92, 0x00011CA7}, {0x00011CA9, 0x00011CB6}, {0x00011D31, 0x00011D36},+{0x00011D3A, 0x00011D3A}, {0x00011D3C, 0x00011D3D}, {0x00011D3F, 0x00011D45}, {0x00011D47, 0x00011D47},+{0x00011D8A, 0x00011D8E}, {0x00011D90, 0x00011D91}, {0x00011D93, 0x00011D97}, {0x00011EF3, 0x00011EF6},+{0x00016AF0, 0x00016AF4}, {0x00016B30, 0x00016B36}, {0x00016F4F, 0x00016F4F}, {0x00016F51, 0x00016F87},+{0x00016F8F, 0x00016F92}, {0x00016FE4, 0x00016FE4}, {0x00016FF0, 0x00016FF1}, {0x0001BC9D, 0x0001BC9E},+{0x0001D165, 0x0001D169}, {0x0001D16D, 0x0001D172}, {0x0001D17B, 0x0001D182}, {0x0001D185, 0x0001D18B},+{0x0001D1AA, 0x0001D1AD}, {0x0001D242, 0x0001D244}, {0x0001DA00, 0x0001DA36}, {0x0001DA3B, 0x0001DA6C},+{0x0001DA75, 0x0001DA75}, {0x0001DA84, 0x0001DA84}, {0x0001DA9B, 0x0001DA9F}, {0x0001DAA1, 0x0001DAAF},+{0x0001E000, 0x0001E006}, {0x0001E008, 0x0001E018}, {0x0001E01B, 0x0001E021}, {0x0001E023, 0x0001E024},+{0x0001E026, 0x0001E02A}, {0x0001E130, 0x0001E136}, {0x0001E2EC, 0x0001E2EF}, {0x0001E8D0, 0x0001E8D6},+{0x0001E944, 0x0001E94A}, {0x000E0100, 0x000E01EF},+};++static const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation = {+{0x00000021, 0x00000023}, {0x00000025, 0x0000002A}, {0x0000002C, 0x0000002F}, {0x0000003A, 0x0000003B},+{0x0000003F, 0x00000040}, {0x0000005B, 0x0000005D}, {0x0000005F, 0x0000005F}, {0x0000007B, 0x0000007B},+{0x0000007D, 0x0000007D}, {0x000000A1, 0x000000A1}, {0x000000A7, 0x000000A7}, {0x000000AB, 0x000000AB},+{0x000000B6, 0x000000B7}, {0x000000BB, 0x000000BB}, {0x000000BF, 0x000000BF}, {0x0000037E, 0x0000037E},+{0x00000387, 0x00000387}, {0x0000055A, 0x0000055F}, {0x00000589, 0x0000058A}, {0x000005BE, 0x000005BE},+{0x000005C0, 0x000005C0}, {0x000005C3, 0x000005C3}, {0x000005C6, 0x000005C6}, {0x000005F3, 0x000005F4},+{0x00000609, 0x0000060A}, {0x0000060C, 0x0000060D}, {0x0000061B, 0x0000061B}, {0x0000061E, 0x0000061F},+{0x0000066A, 0x0000066D}, {0x000006D4, 0x000006D4}, {0x00000700, 0x0000070D}, {0x000007F7, 0x000007F9},+{0x00000830, 0x0000083E}, {0x0000085E, 0x0000085E}, {0x00000964, 0x00000965}, {0x00000970, 0x00000970},+{0x000009FD, 0x000009FD}, {0x00000A76, 0x00000A76}, {0x00000AF0, 0x00000AF0}, {0x00000C77, 0x00000C77},+{0x00000C84, 0x00000C84}, {0x00000DF4, 0x00000DF4}, {0x00000E4F, 0x00000E4F}, {0x00000E5A, 0x00000E5B},+{0x00000F04, 0x00000F12}, {0x00000F14, 0x00000F14}, {0x00000F3A, 0x00000F3D}, {0x00000F85, 0x00000F85},+{0x00000FD0, 0x00000FD4}, {0x00000FD9, 0x00000FDA}, {0x0000104A, 0x0000104F}, {0x000010FB, 0x000010FB},+{0x00001360, 0x00001368}, {0x00001400, 0x00001400}, {0x0000166E, 0x0000166E}, {0x0000169B, 0x0000169C},+{0x000016EB, 0x000016ED}, {0x00001735, 0x00001736}, {0x000017D4, 0x000017D6}, {0x000017D8, 0x000017DA},+{0x00001800, 0x0000180A}, {0x00001944, 0x00001945}, {0x00001A1E, 0x00001A1F}, {0x00001AA0, 0x00001AA6},+{0x00001AA8, 0x00001AAD}, {0x00001B5A, 0x00001B60}, {0x00001BFC, 0x00001BFF}, {0x00001C3B, 0x00001C3F},+{0x00001C7E, 0x00001C7F}, {0x00001CC0, 0x00001CC7}, {0x00001CD3, 0x00001CD3}, {0x00002010, 0x00002027},+{0x00002030, 0x00002043}, {0x00002045, 0x00002051}, {0x00002053, 0x0000205E}, {0x0000207D, 0x0000207E},+{0x0000208D, 0x0000208E}, {0x00002308, 0x0000230B}, {0x00002329, 0x0000232A}, {0x00002768, 0x00002775},+{0x000027C5, 0x000027C6}, {0x000027E6, 0x000027EF}, {0x00002983, 0x00002998}, {0x000029D8, 0x000029DB},+{0x000029FC, 0x000029FD}, {0x00002CF9, 0x00002CFC}, {0x00002CFE, 0x00002CFF}, {0x00002D70, 0x00002D70},+{0x00002E00, 0x00002E2E}, {0x00002E30, 0x00002E4F}, {0x00002E52, 0x00002E52}, {0x00003001, 0x00003003},+{0x00003008, 0x00003011}, {0x00003014, 0x0000301F}, {0x00003030, 0x00003030}, {0x0000303D, 0x0000303D},+{0x000030A0, 0x000030A0}, {0x000030FB, 0x000030FB}, {0x0000A4FE, 0x0000A4FF}, {0x0000A60D, 0x0000A60F},+{0x0000A673, 0x0000A673}, {0x0000A67E, 0x0000A67E}, {0x0000A6F2, 0x0000A6F7}, {0x0000A874, 0x0000A877},+{0x0000A8CE, 0x0000A8CF}, {0x0000A8F8, 0x0000A8FA}, {0x0000A8FC, 0x0000A8FC}, {0x0000A92E, 0x0000A92F},+{0x0000A95F, 0x0000A95F}, {0x0000A9C1, 0x0000A9CD}, {0x0000A9DE, 0x0000A9DF}, {0x0000AA5C, 0x0000AA5F},+{0x0000AADE, 0x0000AADF}, {0x0000AAF0, 0x0000AAF1}, {0x0000ABEB, 0x0000ABEB}, {0x0000FD3E, 0x0000FD3F},+{0x0000FE10, 0x0000FE19}, {0x0000FE30, 0x0000FE52}, {0x0000FE54, 0x0000FE61}, {0x0000FE63, 0x0000FE63},+{0x0000FE68, 0x0000FE68}, {0x0000FE6A, 0x0000FE6B}, {0x0000FF01, 0x0000FF03}, {0x0000FF05, 0x0000FF0A},+{0x0000FF0C, 0x0000FF0F}, {0x0000FF1A, 0x0000FF1B}, {0x0000FF1F, 0x0000FF20}, {0x0000FF3B, 0x0000FF3D},+{0x0000FF3F, 0x0000FF3F}, {0x0000FF5B, 0x0000FF5B}, {0x0000FF5D, 0x0000FF5D}, {0x0000FF5F, 0x0000FF65},+{0x00010100, 0x00010102}, {0x0001039F, 0x0001039F}, {0x000103D0, 0x000103D0}, {0x0001056F, 0x0001056F},+{0x00010857, 0x00010857}, {0x0001091F, 0x0001091F}, {0x0001093F, 0x0001093F}, {0x00010A50, 0x00010A58},+{0x00010A7F, 0x00010A7F}, {0x00010AF0, 0x00010AF6}, {0x00010B39, 0x00010B3F}, {0x00010B99, 0x00010B9C},+{0x00010EAD, 0x00010EAD}, {0x00010F55, 0x00010F59}, {0x00011047, 0x0001104D}, {0x000110BB, 0x000110BC},+{0x000110BE, 0x000110C1}, {0x00011140, 0x00011143}, {0x00011174, 0x00011175}, {0x000111C5, 0x000111C8},+{0x000111CD, 0x000111CD}, {0x000111DB, 0x000111DB}, {0x000111DD, 0x000111DF}, {0x00011238, 0x0001123D},+{0x000112A9, 0x000112A9}, {0x0001144B, 0x0001144F}, {0x0001145A, 0x0001145B}, {0x0001145D, 0x0001145D},+{0x000114C6, 0x000114C6}, {0x000115C1, 0x000115D7}, {0x00011641, 0x00011643}, {0x00011660, 0x0001166C},+{0x0001173C, 0x0001173E}, {0x0001183B, 0x0001183B}, {0x00011944, 0x00011946}, {0x000119E2, 0x000119E2},+{0x00011A3F, 0x00011A46}, {0x00011A9A, 0x00011A9C}, {0x00011A9E, 0x00011AA2}, {0x00011C41, 0x00011C45},+{0x00011C70, 0x00011C71}, {0x00011EF7, 0x00011EF8}, {0x00011FFF, 0x00011FFF}, {0x00012470, 0x00012474},+{0x00016A6E, 0x00016A6F}, {0x00016AF5, 0x00016AF5}, {0x00016B37, 0x00016B3B}, {0x00016B44, 0x00016B44},+{0x00016E97, 0x00016E9A}, {0x00016FE2, 0x00016FE2}, {0x0001BC9F, 0x0001BC9F}, {0x0001DA87, 0x0001DA8B},+{0x0001E95E, 0x0001E95F},+};++static const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol = {+{0x00000024, 0x00000024}, {0x0000002B, 0x0000002B}, {0x0000003C, 0x0000003E}, {0x0000005E, 0x0000005E},+{0x00000060, 0x00000060}, {0x0000007C, 0x0000007C}, {0x0000007E, 0x0000007E}, {0x000000A2, 0x000000A6},+{0x000000A8, 0x000000A9}, {0x000000AC, 0x000000AC}, {0x000000AE, 0x000000B1}, {0x000000B4, 0x000000B4},+{0x000000B8, 0x000000B8}, {0x000000D7, 0x000000D7}, {0x000000F7, 0x000000F7}, {0x000002C2, 0x000002C5},+{0x000002D2, 0x000002DF}, {0x000002E5, 0x000002EB}, {0x000002ED, 0x000002ED}, {0x000002EF, 0x000002FF},+{0x00000375, 0x00000375}, {0x00000384, 0x00000385}, {0x000003F6, 0x000003F6}, {0x00000482, 0x00000482},+{0x0000058D, 0x0000058F}, {0x00000606, 0x00000608}, {0x0000060B, 0x0000060B}, {0x0000060E, 0x0000060F},+{0x000006DE, 0x000006DE}, {0x000006E9, 0x000006E9}, {0x000006FD, 0x000006FE}, {0x000007F6, 0x000007F6},+{0x000007FE, 0x000007FF}, {0x000009F2, 0x000009F3}, {0x000009FA, 0x000009FB}, {0x00000AF1, 0x00000AF1},+{0x00000B70, 0x00000B70}, {0x00000BF3, 0x00000BFA}, {0x00000C7F, 0x00000C7F}, {0x00000D4F, 0x00000D4F},+{0x00000D79, 0x00000D79}, {0x00000E3F, 0x00000E3F}, {0x00000F01, 0x00000F03}, {0x00000F13, 0x00000F13},+{0x00000F15, 0x00000F17}, {0x00000F1A, 0x00000F1F}, {0x00000F34, 0x00000F34}, {0x00000F36, 0x00000F36},+{0x00000F38, 0x00000F38}, {0x00000FBE, 0x00000FC5}, {0x00000FC7, 0x00000FCC}, {0x00000FCE, 0x00000FCF},+{0x00000FD5, 0x00000FD8}, {0x0000109E, 0x0000109F}, {0x00001390, 0x00001399}, {0x0000166D, 0x0000166D},+{0x000017DB, 0x000017DB}, {0x00001940, 0x00001940}, {0x000019DE, 0x000019FF}, {0x00001B61, 0x00001B6A},+{0x00001B74, 0x00001B7C}, {0x00001FBD, 0x00001FBD}, {0x00001FBF, 0x00001FC1}, {0x00001FCD, 0x00001FCF},+{0x00001FDD, 0x00001FDF}, {0x00001FED, 0x00001FEF}, {0x00001FFD, 0x00001FFE}, {0x00002044, 0x00002044},+{0x00002052, 0x00002052}, {0x0000207A, 0x0000207C}, {0x0000208A, 0x0000208C}, {0x000020A0, 0x000020BF},+{0x00002100, 0x00002101}, {0x00002103, 0x00002106}, {0x00002108, 0x00002109}, {0x00002114, 0x00002114},+{0x00002116, 0x00002118}, {0x0000211E, 0x00002123}, {0x00002125, 0x00002125}, {0x00002127, 0x00002127},+{0x00002129, 0x00002129}, {0x0000212E, 0x0000212E}, {0x0000213A, 0x0000213B}, {0x00002140, 0x00002144},+{0x0000214A, 0x0000214D}, {0x0000214F, 0x0000214F}, {0x0000218A, 0x0000218B}, {0x00002190, 0x00002307},+{0x0000230C, 0x00002328}, {0x0000232B, 0x00002426}, {0x00002440, 0x0000244A}, {0x0000249C, 0x000024E9},+{0x00002500, 0x00002767}, {0x00002794, 0x000027C4}, {0x000027C7, 0x000027E5}, {0x000027F0, 0x00002982},+{0x00002999, 0x000029D7}, {0x000029DC, 0x000029FB}, {0x000029FE, 0x00002B73}, {0x00002B76, 0x00002B95},+{0x00002B97, 0x00002BFF}, {0x00002CE5, 0x00002CEA}, {0x00002E50, 0x00002E51}, {0x00002E80, 0x00002E99},+{0x00002E9B, 0x00002EF3}, {0x00002F00, 0x00002FD5}, {0x00002FF0, 0x00002FFB}, {0x00003004, 0x00003004},+{0x00003012, 0x00003013}, {0x00003020, 0x00003020}, {0x00003036, 0x00003037}, {0x0000303E, 0x0000303F},+{0x0000309B, 0x0000309C}, {0x00003190, 0x00003191}, {0x00003196, 0x0000319F}, {0x000031C0, 0x000031E3},+{0x00003200, 0x0000321E}, {0x0000322A, 0x00003247}, {0x00003250, 0x00003250}, {0x00003260, 0x0000327F},+{0x0000328A, 0x000032B0}, {0x000032C0, 0x000033FF}, {0x00004DC0, 0x00004DFF}, {0x0000A490, 0x0000A4C6},+{0x0000A700, 0x0000A716}, {0x0000A720, 0x0000A721}, {0x0000A789, 0x0000A78A}, {0x0000A828, 0x0000A82B},+{0x0000A836, 0x0000A839}, {0x0000AA77, 0x0000AA79}, {0x0000AB5B, 0x0000AB5B}, {0x0000AB6A, 0x0000AB6B},+{0x0000FB29, 0x0000FB29}, {0x0000FBB2, 0x0000FBC1}, {0x0000FDFC, 0x0000FDFD}, {0x0000FE62, 0x0000FE62},+{0x0000FE64, 0x0000FE66}, {0x0000FE69, 0x0000FE69}, {0x0000FF04, 0x0000FF04}, {0x0000FF0B, 0x0000FF0B},+{0x0000FF1C, 0x0000FF1E}, {0x0000FF3E, 0x0000FF3E}, {0x0000FF40, 0x0000FF40}, {0x0000FF5C, 0x0000FF5C},+{0x0000FF5E, 0x0000FF5E}, {0x0000FFE0, 0x0000FFE6}, {0x0000FFE8, 0x0000FFEE}, {0x0000FFFC, 0x0000FFFD},+{0x00010137, 0x0001013F}, {0x00010179, 0x00010189}, {0x0001018C, 0x0001018E}, {0x00010190, 0x0001019C},+{0x000101A0, 0x000101A0}, {0x000101D0, 0x000101FC}, {0x00010877, 0x00010878}, {0x00010AC8, 0x00010AC8},+{0x0001173F, 0x0001173F}, {0x00011FD5, 0x00011FF1}, {0x00016B3C, 0x00016B3F}, {0x00016B45, 0x00016B45},+{0x0001BC9C, 0x0001BC9C}, {0x0001D000, 0x0001D0F5}, {0x0001D100, 0x0001D126}, {0x0001D129, 0x0001D164},+{0x0001D16A, 0x0001D16C}, {0x0001D183, 0x0001D184}, {0x0001D18C, 0x0001D1A9}, {0x0001D1AE, 0x0001D1E8},+{0x0001D200, 0x0001D241}, {0x0001D245, 0x0001D245}, {0x0001D300, 0x0001D356}, {0x0001D6C1, 0x0001D6C1},+{0x0001D6DB, 0x0001D6DB}, {0x0001D6FB, 0x0001D6FB}, {0x0001D715, 0x0001D715}, {0x0001D735, 0x0001D735},+{0x0001D74F, 0x0001D74F}, {0x0001D76F, 0x0001D76F}, {0x0001D789, 0x0001D789}, {0x0001D7A9, 0x0001D7A9},+{0x0001D7C3, 0x0001D7C3}, {0x0001D800, 0x0001D9FF}, {0x0001DA37, 0x0001DA3A}, {0x0001DA6D, 0x0001DA74},+{0x0001DA76, 0x0001DA83}, {0x0001DA85, 0x0001DA86}, {0x0001E14F, 0x0001E14F}, {0x0001E2FF, 0x0001E2FF},+{0x0001ECAC, 0x0001ECAC}, {0x0001ECB0, 0x0001ECB0}, {0x0001ED2E, 0x0001ED2E}, {0x0001EEF0, 0x0001EEF1},+{0x0001F000, 0x0001F02B}, {0x0001F030, 0x0001F093}, {0x0001F0A0, 0x0001F0AE}, {0x0001F0B1, 0x0001F0BF},+{0x0001F0C1, 0x0001F0CF}, {0x0001F0D1, 0x0001F0F5}, {0x0001F10D, 0x0001F1AD}, {0x0001F1E6, 0x0001F202},+{0x0001F210, 0x0001F23B}, {0x0001F240, 0x0001F248}, {0x0001F250, 0x0001F251}, {0x0001F260, 0x0001F265},+{0x0001F300, 0x0001F6D7}, {0x0001F6E0, 0x0001F6EC}, {0x0001F6F0, 0x0001F6FC}, {0x0001F700, 0x0001F773},+{0x0001F780, 0x0001F7D8}, {0x0001F7E0, 0x0001F7EB}, {0x0001F800, 0x0001F80B}, {0x0001F810, 0x0001F847},+{0x0001F850, 0x0001F859}, {0x0001F860, 0x0001F887}, {0x0001F890, 0x0001F8AD}, {0x0001F8B0, 0x0001F8B1},+{0x0001F900, 0x0001F978}, {0x0001F97A, 0x0001F9CB}, {0x0001F9CD, 0x0001FA53}, {0x0001FA60, 0x0001FA6D},+{0x0001FA70, 0x0001FA74}, {0x0001FA78, 0x0001FA7A}, {0x0001FA80, 0x0001FA86}, {0x0001FA90, 0x0001FAA8},+{0x0001FAB0, 0x0001FAB6}, {0x0001FAC0, 0x0001FAC2}, {0x0001FAD0, 0x0001FAD6}, {0x0001FB00, 0x0001FB92},+{0x0001FB94, 0x0001FBCA},+};++static const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control = {+{0x00000000, 0x00000008}, {0x0000000E, 0x0000001B}, {0x0000007F, 0x00000084}, {0x00000086, 0x0000009F},+{0x000000AD, 0x000000AD}, {0x00000378, 0x00000379}, {0x00000380, 0x00000383}, {0x0000038B, 0x0000038B},+{0x0000038D, 0x0000038D}, {0x000003A2, 0x000003A2}, {0x00000530, 0x00000530}, {0x00000557, 0x00000558},+{0x0000058B, 0x0000058C}, {0x00000590, 0x00000590}, {0x000005C8, 0x000005CF}, {0x000005EB, 0x000005EE},+{0x000005F5, 0x00000605}, {0x0000061C, 0x0000061D}, {0x000006DD, 0x000006DD}, {0x0000070E, 0x0000070F},+{0x0000074B, 0x0000074C}, {0x000007B2, 0x000007BF}, {0x000007FB, 0x000007FC}, {0x0000082E, 0x0000082F},+{0x0000083F, 0x0000083F}, {0x0000085C, 0x0000085D}, {0x0000085F, 0x0000085F}, {0x0000086B, 0x0000089F},+{0x000008B5, 0x000008B5}, {0x000008C8, 0x000008D2}, {0x000008E2, 0x000008E2}, {0x00000984, 0x00000984},+{0x0000098D, 0x0000098E}, {0x00000991, 0x00000992}, {0x000009A9, 0x000009A9}, {0x000009B1, 0x000009B1},+{0x000009B3, 0x000009B5}, {0x000009BA, 0x000009BB}, {0x000009C5, 0x000009C6}, {0x000009C9, 0x000009CA},+{0x000009CF, 0x000009D6}, {0x000009D8, 0x000009DB}, {0x000009DE, 0x000009DE}, {0x000009E4, 0x000009E5},+{0x000009FF, 0x00000A00}, {0x00000A04, 0x00000A04}, {0x00000A0B, 0x00000A0E}, {0x00000A11, 0x00000A12},+{0x00000A29, 0x00000A29}, {0x00000A31, 0x00000A31}, {0x00000A34, 0x00000A34}, {0x00000A37, 0x00000A37},+{0x00000A3A, 0x00000A3B}, {0x00000A3D, 0x00000A3D}, {0x00000A43, 0x00000A46}, {0x00000A49, 0x00000A4A},+{0x00000A4E, 0x00000A50}, {0x00000A52, 0x00000A58}, {0x00000A5D, 0x00000A5D}, {0x00000A5F, 0x00000A65},+{0x00000A77, 0x00000A80}, {0x00000A84, 0x00000A84}, {0x00000A8E, 0x00000A8E}, {0x00000A92, 0x00000A92},+{0x00000AA9, 0x00000AA9}, {0x00000AB1, 0x00000AB1}, {0x00000AB4, 0x00000AB4}, {0x00000ABA, 0x00000ABB},+{0x00000AC6, 0x00000AC6}, {0x00000ACA, 0x00000ACA}, {0x00000ACE, 0x00000ACF}, {0x00000AD1, 0x00000ADF},+{0x00000AE4, 0x00000AE5}, {0x00000AF2, 0x00000AF8}, {0x00000B00, 0x00000B00}, {0x00000B04, 0x00000B04},+{0x00000B0D, 0x00000B0E}, {0x00000B11, 0x00000B12}, {0x00000B29, 0x00000B29}, {0x00000B31, 0x00000B31},+{0x00000B34, 0x00000B34}, {0x00000B3A, 0x00000B3B}, {0x00000B45, 0x00000B46}, {0x00000B49, 0x00000B4A},+{0x00000B4E, 0x00000B54}, {0x00000B58, 0x00000B5B}, {0x00000B5E, 0x00000B5E}, {0x00000B64, 0x00000B65},+{0x00000B78, 0x00000B81}, {0x00000B84, 0x00000B84}, {0x00000B8B, 0x00000B8D}, {0x00000B91, 0x00000B91},+{0x00000B96, 0x00000B98}, {0x00000B9B, 0x00000B9B}, {0x00000B9D, 0x00000B9D}, {0x00000BA0, 0x00000BA2},+{0x00000BA5, 0x00000BA7}, {0x00000BAB, 0x00000BAD}, {0x00000BBA, 0x00000BBD}, {0x00000BC3, 0x00000BC5},+{0x00000BC9, 0x00000BC9}, {0x00000BCE, 0x00000BCF}, {0x00000BD1, 0x00000BD6}, {0x00000BD8, 0x00000BE5},+{0x00000BFB, 0x00000BFF}, {0x00000C0D, 0x00000C0D}, {0x00000C11, 0x00000C11}, {0x00000C29, 0x00000C29},+{0x00000C3A, 0x00000C3C}, {0x00000C45, 0x00000C45}, {0x00000C49, 0x00000C49}, {0x00000C4E, 0x00000C54},+{0x00000C57, 0x00000C57}, {0x00000C5B, 0x00000C5F}, {0x00000C64, 0x00000C65}, {0x00000C70, 0x00000C76},+{0x00000C8D, 0x00000C8D}, {0x00000C91, 0x00000C91}, {0x00000CA9, 0x00000CA9}, {0x00000CB4, 0x00000CB4},+{0x00000CBA, 0x00000CBB}, {0x00000CC5, 0x00000CC5}, {0x00000CC9, 0x00000CC9}, {0x00000CCE, 0x00000CD4},+{0x00000CD7, 0x00000CDD}, {0x00000CDF, 0x00000CDF}, {0x00000CE4, 0x00000CE5}, {0x00000CF0, 0x00000CF0},+{0x00000CF3, 0x00000CFF}, {0x00000D0D, 0x00000D0D}, {0x00000D11, 0x00000D11}, {0x00000D45, 0x00000D45},+{0x00000D49, 0x00000D49}, {0x00000D50, 0x00000D53}, {0x00000D64, 0x00000D65}, {0x00000D80, 0x00000D80},+{0x00000D84, 0x00000D84}, {0x00000D97, 0x00000D99}, {0x00000DB2, 0x00000DB2}, {0x00000DBC, 0x00000DBC},+{0x00000DBE, 0x00000DBF}, {0x00000DC7, 0x00000DC9}, {0x00000DCB, 0x00000DCE}, {0x00000DD5, 0x00000DD5},+{0x00000DD7, 0x00000DD7}, {0x00000DE0, 0x00000DE5}, {0x00000DF0, 0x00000DF1}, {0x00000DF5, 0x00000E00},+{0x00000E3B, 0x00000E3E}, {0x00000E5C, 0x00000E80}, {0x00000E83, 0x00000E83}, {0x00000E85, 0x00000E85},+{0x00000E8B, 0x00000E8B}, {0x00000EA4, 0x00000EA4}, {0x00000EA6, 0x00000EA6}, {0x00000EBE, 0x00000EBF},+{0x00000EC5, 0x00000EC5}, {0x00000EC7, 0x00000EC7}, {0x00000ECE, 0x00000ECF}, {0x00000EDA, 0x00000EDB},+{0x00000EE0, 0x00000EFF}, {0x00000F48, 0x00000F48}, {0x00000F6D, 0x00000F70}, {0x00000F98, 0x00000F98},+{0x00000FBD, 0x00000FBD}, {0x00000FCD, 0x00000FCD}, {0x00000FDB, 0x00000FFF}, {0x000010C6, 0x000010C6},+{0x000010C8, 0x000010CC}, {0x000010CE, 0x000010CF}, {0x00001249, 0x00001249}, {0x0000124E, 0x0000124F},+{0x00001257, 0x00001257}, {0x00001259, 0x00001259}, {0x0000125E, 0x0000125F}, {0x00001289, 0x00001289},+{0x0000128E, 0x0000128F}, {0x000012B1, 0x000012B1}, {0x000012B6, 0x000012B7}, {0x000012BF, 0x000012BF},+{0x000012C1, 0x000012C1}, {0x000012C6, 0x000012C7}, {0x000012D7, 0x000012D7}, {0x00001311, 0x00001311},+{0x00001316, 0x00001317}, {0x0000135B, 0x0000135C}, {0x0000137D, 0x0000137F}, {0x0000139A, 0x0000139F},+{0x000013F6, 0x000013F7}, {0x000013FE, 0x000013FF}, {0x0000169D, 0x0000169F}, {0x000016F9, 0x000016FF},+{0x0000170D, 0x0000170D}, {0x00001715, 0x0000171F}, {0x00001737, 0x0000173F}, {0x00001754, 0x0000175F},+{0x0000176D, 0x0000176D}, {0x00001771, 0x00001771}, {0x00001774, 0x0000177F}, {0x000017DE, 0x000017DF},+{0x000017EA, 0x000017EF}, {0x000017FA, 0x000017FF}, {0x0000180E, 0x0000180F}, {0x0000181A, 0x0000181F},+{0x00001879, 0x0000187F}, {0x000018AB, 0x000018AF}, {0x000018F6, 0x000018FF}, {0x0000191F, 0x0000191F},+{0x0000192C, 0x0000192F}, {0x0000193C, 0x0000193F}, {0x00001941, 0x00001943}, {0x0000196E, 0x0000196F},+{0x00001975, 0x0000197F}, {0x000019AC, 0x000019AF}, {0x000019CA, 0x000019CF}, {0x000019DB, 0x000019DD},+{0x00001A1C, 0x00001A1D}, {0x00001A5F, 0x00001A5F}, {0x00001A7D, 0x00001A7E}, {0x00001A8A, 0x00001A8F},+{0x00001A9A, 0x00001A9F}, {0x00001AAE, 0x00001AAF}, {0x00001AC1, 0x00001AFF}, {0x00001B4C, 0x00001B4F},+{0x00001B7D, 0x00001B7F}, {0x00001BF4, 0x00001BFB}, {0x00001C38, 0x00001C3A}, {0x00001C4A, 0x00001C4C},+{0x00001C89, 0x00001C8F}, {0x00001CBB, 0x00001CBC}, {0x00001CC8, 0x00001CCF}, {0x00001CFB, 0x00001CFF},+{0x00001DFA, 0x00001DFA}, {0x00001F16, 0x00001F17}, {0x00001F1E, 0x00001F1F}, {0x00001F46, 0x00001F47},+{0x00001F4E, 0x00001F4F}, {0x00001F58, 0x00001F58}, {0x00001F5A, 0x00001F5A}, {0x00001F5C, 0x00001F5C},+{0x00001F5E, 0x00001F5E}, {0x00001F7E, 0x00001F7F}, {0x00001FB5, 0x00001FB5}, {0x00001FC5, 0x00001FC5},+{0x00001FD4, 0x00001FD5}, {0x00001FDC, 0x00001FDC}, {0x00001FF0, 0x00001FF1}, {0x00001FF5, 0x00001FF5},+{0x00001FFF, 0x00001FFF}, {0x0000200B, 0x0000200F}, {0x0000202A, 0x0000202E}, {0x00002060, 0x0000206F},+{0x00002072, 0x00002073}, {0x0000208F, 0x0000208F}, {0x0000209D, 0x0000209F}, {0x000020C0, 0x000020CF},+{0x000020F1, 0x000020FF}, {0x0000218C, 0x0000218F}, {0x00002427, 0x0000243F}, {0x0000244B, 0x0000245F},+{0x00002B74, 0x00002B75}, {0x00002B96, 0x00002B96}, {0x00002C2F, 0x00002C2F}, {0x00002C5F, 0x00002C5F},+{0x00002CF4, 0x00002CF8}, {0x00002D26, 0x00002D26}, {0x00002D28, 0x00002D2C}, {0x00002D2E, 0x00002D2F},+{0x00002D68, 0x00002D6E}, {0x00002D71, 0x00002D7E}, {0x00002D97, 0x00002D9F}, {0x00002DA7, 0x00002DA7},+{0x00002DAF, 0x00002DAF}, {0x00002DB7, 0x00002DB7}, {0x00002DBF, 0x00002DBF}, {0x00002DC7, 0x00002DC7},+{0x00002DCF, 0x00002DCF}, {0x00002DD7, 0x00002DD7}, {0x00002DDF, 0x00002DDF}, {0x00002E53, 0x00002E7F},+{0x00002E9A, 0x00002E9A}, {0x00002EF4, 0x00002EFF}, {0x00002FD6, 0x00002FEF}, {0x00002FFC, 0x00002FFF},+{0x00003040, 0x00003040}, {0x00003097, 0x00003098}, {0x00003100, 0x00003104}, {0x00003130, 0x00003130},+{0x0000318F, 0x0000318F}, {0x000031E4, 0x000031EF}, {0x0000321F, 0x0000321F}, {0x00009FFD, 0x00009FFF},+{0x0000A48D, 0x0000A48F}, {0x0000A4C7, 0x0000A4CF}, {0x0000A62C, 0x0000A63F}, {0x0000A6F8, 0x0000A6FF},+{0x0000A7C0, 0x0000A7C1}, {0x0000A7CB, 0x0000A7F4}, {0x0000A82D, 0x0000A82F}, {0x0000A83A, 0x0000A83F},+{0x0000A878, 0x0000A87F}, {0x0000A8C6, 0x0000A8CD}, {0x0000A8DA, 0x0000A8DF}, {0x0000A954, 0x0000A95E},+{0x0000A97D, 0x0000A97F}, {0x0000A9CE, 0x0000A9CE}, {0x0000A9DA, 0x0000A9DD}, {0x0000A9FF, 0x0000A9FF},+{0x0000AA37, 0x0000AA3F}, {0x0000AA4E, 0x0000AA4F}, {0x0000AA5A, 0x0000AA5B}, {0x0000AAC3, 0x0000AADA},+{0x0000AAF7, 0x0000AB00}, {0x0000AB07, 0x0000AB08}, {0x0000AB0F, 0x0000AB10}, {0x0000AB17, 0x0000AB1F},+{0x0000AB27, 0x0000AB27}, {0x0000AB2F, 0x0000AB2F}, {0x0000AB6C, 0x0000AB6F}, {0x0000ABEE, 0x0000ABEF},+{0x0000ABFA, 0x0000ABFF}, {0x0000D7A4, 0x0000D7AF}, {0x0000D7C7, 0x0000D7CA}, {0x0000D7FC, 0x0000F8FF},+{0x0000FA6E, 0x0000FA6F}, {0x0000FADA, 0x0000FAFF}, {0x0000FB07, 0x0000FB12}, {0x0000FB18, 0x0000FB1C},+{0x0000FB37, 0x0000FB37}, {0x0000FB3D, 0x0000FB3D}, {0x0000FB3F, 0x0000FB3F}, {0x0000FB42, 0x0000FB42},+{0x0000FB45, 0x0000FB45}, {0x0000FBC2, 0x0000FBD2}, {0x0000FD40, 0x0000FD4F}, {0x0000FD90, 0x0000FD91},+{0x0000FDC8, 0x0000FDEF}, {0x0000FDFE, 0x0000FDFF}, {0x0000FE1A, 0x0000FE1F}, {0x0000FE53, 0x0000FE53},+{0x0000FE67, 0x0000FE67}, {0x0000FE6C, 0x0000FE6F}, {0x0000FE75, 0x0000FE75}, {0x0000FEFD, 0x0000FF00},+{0x0000FFBF, 0x0000FFC1}, {0x0000FFC8, 0x0000FFC9}, {0x0000FFD0, 0x0000FFD1}, {0x0000FFD8, 0x0000FFD9},+{0x0000FFDD, 0x0000FFDF}, {0x0000FFE7, 0x0000FFE7}, {0x0000FFEF, 0x0000FFFB}, {0x0000FFFE, 0x0000FFFF},+{0x0001000C, 0x0001000C}, {0x00010027, 0x00010027}, {0x0001003B, 0x0001003B}, {0x0001003E, 0x0001003E},+{0x0001004E, 0x0001004F}, {0x0001005E, 0x0001007F}, {0x000100FB, 0x000100FF}, {0x00010103, 0x00010106},+{0x00010134, 0x00010136}, {0x0001018F, 0x0001018F}, {0x0001019D, 0x0001019F}, {0x000101A1, 0x000101CF},+{0x000101FE, 0x0001027F}, {0x0001029D, 0x0001029F}, {0x000102D1, 0x000102DF}, {0x000102FC, 0x000102FF},+{0x00010324, 0x0001032C}, {0x0001034B, 0x0001034F}, {0x0001037B, 0x0001037F}, {0x0001039E, 0x0001039E},+{0x000103C4, 0x000103C7}, {0x000103D6, 0x000103FF}, {0x0001049E, 0x0001049F}, {0x000104AA, 0x000104AF},+{0x000104D4, 0x000104D7}, {0x000104FC, 0x000104FF}, {0x00010528, 0x0001052F}, {0x00010564, 0x0001056E},+{0x00010570, 0x000105FF}, {0x00010737, 0x0001073F}, {0x00010756, 0x0001075F}, {0x00010768, 0x000107FF},+{0x00010806, 0x00010807}, {0x00010809, 0x00010809}, {0x00010836, 0x00010836}, {0x00010839, 0x0001083B},+{0x0001083D, 0x0001083E}, {0x00010856, 0x00010856}, {0x0001089F, 0x000108A6}, {0x000108B0, 0x000108DF},+{0x000108F3, 0x000108F3}, {0x000108F6, 0x000108FA}, {0x0001091C, 0x0001091E}, {0x0001093A, 0x0001093E},+{0x00010940, 0x0001097F}, {0x000109B8, 0x000109BB}, {0x000109D0, 0x000109D1}, {0x00010A04, 0x00010A04},+{0x00010A07, 0x00010A0B}, {0x00010A14, 0x00010A14}, {0x00010A18, 0x00010A18}, {0x00010A36, 0x00010A37},+{0x00010A3B, 0x00010A3E}, {0x00010A49, 0x00010A4F}, {0x00010A59, 0x00010A5F}, {0x00010AA0, 0x00010ABF},+{0x00010AE7, 0x00010AEA}, {0x00010AF7, 0x00010AFF}, {0x00010B36, 0x00010B38}, {0x00010B56, 0x00010B57},+{0x00010B73, 0x00010B77}, {0x00010B92, 0x00010B98}, {0x00010B9D, 0x00010BA8}, {0x00010BB0, 0x00010BFF},+{0x00010C49, 0x00010C7F}, {0x00010CB3, 0x00010CBF}, {0x00010CF3, 0x00010CF9}, {0x00010D28, 0x00010D2F},+{0x00010D3A, 0x00010E5F}, {0x00010E7F, 0x00010E7F}, {0x00010EAA, 0x00010EAA}, {0x00010EAE, 0x00010EAF},+{0x00010EB2, 0x00010EFF}, {0x00010F28, 0x00010F2F}, {0x00010F5A, 0x00010FAF}, {0x00010FCC, 0x00010FDF},+{0x00010FF7, 0x00010FFF}, {0x0001104E, 0x00011051}, {0x00011070, 0x0001107E}, {0x000110BD, 0x000110BD},+{0x000110C2, 0x000110CF}, {0x000110E9, 0x000110EF}, {0x000110FA, 0x000110FF}, {0x00011135, 0x00011135},+{0x00011148, 0x0001114F}, {0x00011177, 0x0001117F}, {0x000111E0, 0x000111E0}, {0x000111F5, 0x000111FF},+{0x00011212, 0x00011212}, {0x0001123F, 0x0001127F}, {0x00011287, 0x00011287}, {0x00011289, 0x00011289},+{0x0001128E, 0x0001128E}, {0x0001129E, 0x0001129E}, {0x000112AA, 0x000112AF}, {0x000112EB, 0x000112EF},+{0x000112FA, 0x000112FF}, {0x00011304, 0x00011304}, {0x0001130D, 0x0001130E}, {0x00011311, 0x00011312},+{0x00011329, 0x00011329}, {0x00011331, 0x00011331}, {0x00011334, 0x00011334}, {0x0001133A, 0x0001133A},+{0x00011345, 0x00011346}, {0x00011349, 0x0001134A}, {0x0001134E, 0x0001134F}, {0x00011351, 0x00011356},+{0x00011358, 0x0001135C}, {0x00011364, 0x00011365}, {0x0001136D, 0x0001136F}, {0x00011375, 0x000113FF},+{0x0001145C, 0x0001145C}, {0x00011462, 0x0001147F}, {0x000114C8, 0x000114CF}, {0x000114DA, 0x0001157F},+{0x000115B6, 0x000115B7}, {0x000115DE, 0x000115FF}, {0x00011645, 0x0001164F}, {0x0001165A, 0x0001165F},+{0x0001166D, 0x0001167F}, {0x000116B9, 0x000116BF}, {0x000116CA, 0x000116FF}, {0x0001171B, 0x0001171C},+{0x0001172C, 0x0001172F}, {0x00011740, 0x000117FF}, {0x0001183C, 0x0001189F}, {0x000118F3, 0x000118FE},+{0x00011907, 0x00011908}, {0x0001190A, 0x0001190B}, {0x00011914, 0x00011914}, {0x00011917, 0x00011917},+{0x00011936, 0x00011936}, {0x00011939, 0x0001193A}, {0x00011947, 0x0001194F}, {0x0001195A, 0x0001199F},+{0x000119A8, 0x000119A9}, {0x000119D8, 0x000119D9}, {0x000119E5, 0x000119FF}, {0x00011A48, 0x00011A4F},+{0x00011AA3, 0x00011ABF}, {0x00011AF9, 0x00011BFF}, {0x00011C09, 0x00011C09}, {0x00011C37, 0x00011C37},+{0x00011C46, 0x00011C4F}, {0x00011C6D, 0x00011C6F}, {0x00011C90, 0x00011C91}, {0x00011CA8, 0x00011CA8},+{0x00011CB7, 0x00011CFF}, {0x00011D07, 0x00011D07}, {0x00011D0A, 0x00011D0A}, {0x00011D37, 0x00011D39},+{0x00011D3B, 0x00011D3B}, {0x00011D3E, 0x00011D3E}, {0x00011D48, 0x00011D4F}, {0x00011D5A, 0x00011D5F},+{0x00011D66, 0x00011D66}, {0x00011D69, 0x00011D69}, {0x00011D8F, 0x00011D8F}, {0x00011D92, 0x00011D92},+{0x00011D99, 0x00011D9F}, {0x00011DAA, 0x00011EDF}, {0x00011EF9, 0x00011FAF}, {0x00011FB1, 0x00011FBF},+{0x00011FF2, 0x00011FFE}, {0x0001239A, 0x000123FF}, {0x0001246F, 0x0001246F}, {0x00012475, 0x0001247F},+{0x00012544, 0x00012FFF}, {0x0001342F, 0x000143FF}, {0x00014647, 0x000167FF}, {0x00016A39, 0x00016A3F},+{0x00016A5F, 0x00016A5F}, {0x00016A6A, 0x00016A6D}, {0x00016A70, 0x00016ACF}, {0x00016AEE, 0x00016AEF},+{0x00016AF6, 0x00016AFF}, {0x00016B46, 0x00016B4F}, {0x00016B5A, 0x00016B5A}, {0x00016B62, 0x00016B62},+{0x00016B78, 0x00016B7C}, {0x00016B90, 0x00016E3F}, {0x00016E9B, 0x00016EFF}, {0x00016F4B, 0x00016F4E},+{0x00016F88, 0x00016F8E}, {0x00016FA0, 0x00016FDF}, {0x00016FE5, 0x00016FEF}, {0x00016FF2, 0x00016FFF},+{0x000187F8, 0x000187FF}, {0x00018CD6, 0x00018CFF}, {0x00018D09, 0x0001AFFF}, {0x0001B11F, 0x0001B14F},+{0x0001B153, 0x0001B163}, {0x0001B168, 0x0001B16F}, {0x0001B2FC, 0x0001BBFF}, {0x0001BC6B, 0x0001BC6F},+{0x0001BC7D, 0x0001BC7F}, {0x0001BC89, 0x0001BC8F}, {0x0001BC9A, 0x0001BC9B}, {0x0001BCA0, 0x0001CFFF},+{0x0001D0F6, 0x0001D0FF}, {0x0001D127, 0x0001D128}, {0x0001D173, 0x0001D17A}, {0x0001D1E9, 0x0001D1FF},+{0x0001D246, 0x0001D2DF}, {0x0001D2F4, 0x0001D2FF}, {0x0001D357, 0x0001D35F}, {0x0001D379, 0x0001D3FF},+{0x0001D455, 0x0001D455}, {0x0001D49D, 0x0001D49D}, {0x0001D4A0, 0x0001D4A1}, {0x0001D4A3, 0x0001D4A4},+{0x0001D4A7, 0x0001D4A8}, {0x0001D4AD, 0x0001D4AD}, {0x0001D4BA, 0x0001D4BA}, {0x0001D4BC, 0x0001D4BC},+{0x0001D4C4, 0x0001D4C4}, {0x0001D506, 0x0001D506}, {0x0001D50B, 0x0001D50C}, {0x0001D515, 0x0001D515},+{0x0001D51D, 0x0001D51D}, {0x0001D53A, 0x0001D53A}, {0x0001D53F, 0x0001D53F}, {0x0001D545, 0x0001D545},+{0x0001D547, 0x0001D549}, {0x0001D551, 0x0001D551}, {0x0001D6A6, 0x0001D6A7}, {0x0001D7CC, 0x0001D7CD},+{0x0001DA8C, 0x0001DA9A}, {0x0001DAA0, 0x0001DAA0}, {0x0001DAB0, 0x0001DFFF}, {0x0001E007, 0x0001E007},+{0x0001E019, 0x0001E01A}, {0x0001E022, 0x0001E022}, {0x0001E025, 0x0001E025}, {0x0001E02B, 0x0001E0FF},+{0x0001E12D, 0x0001E12F}, {0x0001E13E, 0x0001E13F}, {0x0001E14A, 0x0001E14D}, {0x0001E150, 0x0001E2BF},+{0x0001E2FA, 0x0001E2FE}, {0x0001E300, 0x0001E7FF}, {0x0001E8C5, 0x0001E8C6}, {0x0001E8D7, 0x0001E8FF},+{0x0001E94C, 0x0001E94F}, {0x0001E95A, 0x0001E95D}, {0x0001E960, 0x0001EC70}, {0x0001ECB5, 0x0001ED00},+{0x0001ED3E, 0x0001EDFF}, {0x0001EE04, 0x0001EE04}, {0x0001EE20, 0x0001EE20}, {0x0001EE23, 0x0001EE23},+{0x0001EE25, 0x0001EE26}, {0x0001EE28, 0x0001EE28}, {0x0001EE33, 0x0001EE33}, {0x0001EE38, 0x0001EE38},+{0x0001EE3A, 0x0001EE3A}, {0x0001EE3C, 0x0001EE41}, {0x0001EE43, 0x0001EE46}, {0x0001EE48, 0x0001EE48},+{0x0001EE4A, 0x0001EE4A}, {0x0001EE4C, 0x0001EE4C}, {0x0001EE50, 0x0001EE50}, {0x0001EE53, 0x0001EE53},+{0x0001EE55, 0x0001EE56}, {0x0001EE58, 0x0001EE58}, {0x0001EE5A, 0x0001EE5A}, {0x0001EE5C, 0x0001EE5C},+{0x0001EE5E, 0x0001EE5E}, {0x0001EE60, 0x0001EE60}, {0x0001EE63, 0x0001EE63}, {0x0001EE65, 0x0001EE66},+{0x0001EE6B, 0x0001EE6B}, {0x0001EE73, 0x0001EE73}, {0x0001EE78, 0x0001EE78}, {0x0001EE7D, 0x0001EE7D},+{0x0001EE7F, 0x0001EE7F}, {0x0001EE8A, 0x0001EE8A}, {0x0001EE9C, 0x0001EEA0}, {0x0001EEA4, 0x0001EEA4},+{0x0001EEAA, 0x0001EEAA}, {0x0001EEBC, 0x0001EEEF}, {0x0001EEF2, 0x0001EFFF}, {0x0001F02C, 0x0001F02F},+{0x0001F094, 0x0001F09F}, {0x0001F0AF, 0x0001F0B0}, {0x0001F0C0, 0x0001F0C0}, {0x0001F0D0, 0x0001F0D0},+{0x0001F0F6, 0x0001F0FF}, {0x0001F1AE, 0x0001F1E5}, {0x0001F203, 0x0001F20F}, {0x0001F23C, 0x0001F23F},+{0x0001F249, 0x0001F24F}, {0x0001F252, 0x0001F25F}, {0x0001F266, 0x0001F2FF}, {0x0001F6D8, 0x0001F6DF},+{0x0001F6ED, 0x0001F6EF}, {0x0001F6FD, 0x0001F6FF}, {0x0001F774, 0x0001F77F}, {0x0001F7D9, 0x0001F7DF},+{0x0001F7EC, 0x0001F7FF}, {0x0001F80C, 0x0001F80F}, {0x0001F848, 0x0001F84F}, {0x0001F85A, 0x0001F85F},+{0x0001F888, 0x0001F88F}, {0x0001F8AE, 0x0001F8AF}, {0x0001F8B2, 0x0001F8FF}, {0x0001F979, 0x0001F979},+{0x0001F9CC, 0x0001F9CC}, {0x0001FA54, 0x0001FA5F}, {0x0001FA6E, 0x0001FA6F}, {0x0001FA75, 0x0001FA77},+{0x0001FA7B, 0x0001FA7F}, {0x0001FA87, 0x0001FA8F}, {0x0001FAA9, 0x0001FAAF}, {0x0001FAB7, 0x0001FABF},+{0x0001FAC3, 0x0001FACF}, {0x0001FAD7, 0x0001FAFF}, {0x0001FB93, 0x0001FB93}, {0x0001FBCB, 0x0001FBEF},+{0x0001FBFA, 0x0001FFFF}, {0x0002A6DE, 0x0002A6FF}, {0x0002B735, 0x0002B73F}, {0x0002B81E, 0x0002B81F},+{0x0002CEA2, 0x0002CEAF}, {0x0002EBE1, 0x0002F7FF}, {0x0002FA1E, 0x0002FFFF}, {0x0003134B, 0x000E00FF},+{0x000E01F0, 0x0010FFFF},+};++static const std::multimap<uint32_t, uint32_t> unicode_map_nfd = {+{0x000000C0, 0x00000041}, {0x000000C0, 0x00000300}, {0x000000C1, 0x00000041}, {0x000000C1, 0x00000301},+{0x000000C2, 0x00000041}, {0x000000C2, 0x00000302}, {0x000000C3, 0x00000041}, {0x000000C3, 0x00000303},+{0x000000C4, 0x00000041}, {0x000000C4, 0x00000308}, {0x000000C5, 0x00000041}, {0x000000C5, 0x0000030A},+{0x000000C7, 0x00000043}, {0x000000C7, 0x00000327}, {0x000000C8, 0x00000045}, {0x000000C8, 0x00000300},+{0x000000C9, 0x00000045}, {0x000000C9, 0x00000301}, {0x000000CA, 0x00000045}, {0x000000CA, 0x00000302},+{0x000000CB, 0x00000045}, {0x000000CB, 0x00000308}, {0x000000CC, 0x00000049}, {0x000000CC, 0x00000300},+{0x000000CD, 0x00000049}, {0x000000CD, 0x00000301}, {0x000000CE, 0x00000049}, {0x000000CE, 0x00000302},+{0x000000CF, 0x00000049}, {0x000000CF, 0x00000308}, {0x000000D1, 0x0000004E}, {0x000000D1, 0x00000303},+{0x000000D2, 0x0000004F}, {0x000000D2, 0x00000300}, {0x000000D3, 0x0000004F}, {0x000000D3, 0x00000301},+{0x000000D4, 0x0000004F}, {0x000000D4, 0x00000302}, {0x000000D5, 0x0000004F}, {0x000000D5, 0x00000303},+{0x000000D6, 0x0000004F}, {0x000000D6, 0x00000308}, {0x000000D9, 0x00000055}, {0x000000D9, 0x00000300},+{0x000000DA, 0x00000055}, {0x000000DA, 0x00000301}, {0x000000DB, 0x00000055}, {0x000000DB, 0x00000302},+{0x000000DC, 0x00000055}, {0x000000DC, 0x00000308}, {0x000000DD, 0x00000059}, {0x000000DD, 0x00000301},+{0x000000E0, 0x00000061}, {0x000000E0, 0x00000300}, {0x000000E1, 0x00000061}, {0x000000E1, 0x00000301},+{0x000000E2, 0x00000061}, {0x000000E2, 0x00000302}, {0x000000E3, 0x00000061}, {0x000000E3, 0x00000303},+{0x000000E4, 0x00000061}, {0x000000E4, 0x00000308}, {0x000000E5, 0x00000061}, {0x000000E5, 0x0000030A},+{0x000000E7, 0x00000063}, {0x000000E7, 0x00000327}, {0x000000E8, 0x00000065}, {0x000000E8, 0x00000300},+{0x000000E9, 0x00000065}, {0x000000E9, 0x00000301}, {0x000000EA, 0x00000065}, {0x000000EA, 0x00000302},+{0x000000EB, 0x00000065}, {0x000000EB, 0x00000308}, {0x000000EC, 0x00000069}, {0x000000EC, 0x00000300},+{0x000000ED, 0x00000069}, {0x000000ED, 0x00000301}, {0x000000EE, 0x00000069}, {0x000000EE, 0x00000302},+{0x000000EF, 0x00000069}, {0x000000EF, 0x00000308}, {0x000000F1, 0x0000006E}, {0x000000F1, 0x00000303},+{0x000000F2, 0x0000006F}, {0x000000F2, 0x00000300}, {0x000000F3, 0x0000006F}, {0x000000F3, 0x00000301},+{0x000000F4, 0x0000006F}, {0x000000F4, 0x00000302}, {0x000000F5, 0x0000006F}, {0x000000F5, 0x00000303},+{0x000000F6, 0x0000006F}, {0x000000F6, 0x00000308}, {0x000000F9, 0x00000075}, {0x000000F9, 0x00000300},+{0x000000FA, 0x00000075}, {0x000000FA, 0x00000301}, {0x000000FB, 0x00000075}, {0x000000FB, 0x00000302},+{0x000000FC, 0x00000075}, {0x000000FC, 0x00000308}, {0x000000FD, 0x00000079}, {0x000000FD, 0x00000301},+{0x000000FF, 0x00000079}, {0x000000FF, 0x00000308}, {0x00000100, 0x00000041}, {0x00000100, 0x00000304},+{0x00000101, 0x00000061}, {0x00000101, 0x00000304}, {0x00000102, 0x00000041}, {0x00000102, 0x00000306},+{0x00000103, 0x00000061}, {0x00000103, 0x00000306}, {0x00000104, 0x00000041}, {0x00000104, 0x00000328},+{0x00000105, 0x00000061}, {0x00000105, 0x00000328}, {0x00000106, 0x00000043}, {0x00000106, 0x00000301},+{0x00000107, 0x00000063}, {0x00000107, 0x00000301}, {0x00000108, 0x00000043}, {0x00000108, 0x00000302},+{0x00000109, 0x00000063}, {0x00000109, 0x00000302}, {0x0000010A, 0x00000043}, {0x0000010A, 0x00000307},+{0x0000010B, 0x00000063}, {0x0000010B, 0x00000307}, {0x0000010C, 0x00000043}, {0x0000010C, 0x0000030C},+{0x0000010D, 0x00000063}, {0x0000010D, 0x0000030C}, {0x0000010E, 0x00000044}, {0x0000010E, 0x0000030C},+{0x0000010F, 0x00000064}, {0x0000010F, 0x0000030C}, {0x00000112, 0x00000045}, {0x00000112, 0x00000304},+{0x00000113, 0x00000065}, {0x00000113, 0x00000304}, {0x00000114, 0x00000045}, {0x00000114, 0x00000306},+{0x00000115, 0x00000065}, {0x00000115, 0x00000306}, {0x00000116, 0x00000045}, {0x00000116, 0x00000307},+{0x00000117, 0x00000065}, {0x00000117, 0x00000307}, {0x00000118, 0x00000045}, {0x00000118, 0x00000328},+{0x00000119, 0x00000065}, {0x00000119, 0x00000328}, {0x0000011A, 0x00000045}, {0x0000011A, 0x0000030C},+{0x0000011B, 0x00000065}, {0x0000011B, 0x0000030C}, {0x0000011C, 0x00000047}, {0x0000011C, 0x00000302},+{0x0000011D, 0x00000067}, {0x0000011D, 0x00000302}, {0x0000011E, 0x00000047}, {0x0000011E, 0x00000306},+{0x0000011F, 0x00000067}, {0x0000011F, 0x00000306}, {0x00000120, 0x00000047}, {0x00000120, 0x00000307},+{0x00000121, 0x00000067}, {0x00000121, 0x00000307}, {0x00000122, 0x00000047}, {0x00000122, 0x00000327},+{0x00000123, 0x00000067}, {0x00000123, 0x00000327}, {0x00000124, 0x00000048}, {0x00000124, 0x00000302},+{0x00000125, 0x00000068}, {0x00000125, 0x00000302}, {0x00000128, 0x00000049}, {0x00000128, 0x00000303},+{0x00000129, 0x00000069}, {0x00000129, 0x00000303}, {0x0000012A, 0x00000049}, {0x0000012A, 0x00000304},+{0x0000012B, 0x00000069}, {0x0000012B, 0x00000304}, {0x0000012C, 0x00000049}, {0x0000012C, 0x00000306},+{0x0000012D, 0x00000069}, {0x0000012D, 0x00000306}, {0x0000012E, 0x00000049}, {0x0000012E, 0x00000328},+{0x0000012F, 0x00000069}, {0x0000012F, 0x00000328}, {0x00000130, 0x00000049}, {0x00000130, 0x00000307},+{0x00000134, 0x0000004A}, {0x00000134, 0x00000302}, {0x00000135, 0x0000006A}, {0x00000135, 0x00000302},+{0x00000136, 0x0000004B}, {0x00000136, 0x00000327}, {0x00000137, 0x0000006B}, {0x00000137, 0x00000327},+{0x00000139, 0x0000004C}, {0x00000139, 0x00000301}, {0x0000013A, 0x0000006C}, {0x0000013A, 0x00000301},+{0x0000013B, 0x0000004C}, {0x0000013B, 0x00000327}, {0x0000013C, 0x0000006C}, {0x0000013C, 0x00000327},+{0x0000013D, 0x0000004C}, {0x0000013D, 0x0000030C}, {0x0000013E, 0x0000006C}, {0x0000013E, 0x0000030C},+{0x00000143, 0x0000004E}, {0x00000143, 0x00000301}, {0x00000144, 0x0000006E}, {0x00000144, 0x00000301},+{0x00000145, 0x0000004E}, {0x00000145, 0x00000327}, {0x00000146, 0x0000006E}, {0x00000146, 0x00000327},+{0x00000147, 0x0000004E}, {0x00000147, 0x0000030C}, {0x00000148, 0x0000006E}, {0x00000148, 0x0000030C},+{0x0000014C, 0x0000004F}, {0x0000014C, 0x00000304}, {0x0000014D, 0x0000006F}, {0x0000014D, 0x00000304},+{0x0000014E, 0x0000004F}, {0x0000014E, 0x00000306}, {0x0000014F, 0x0000006F}, {0x0000014F, 0x00000306},+{0x00000150, 0x0000004F}, {0x00000150, 0x0000030B}, {0x00000151, 0x0000006F}, {0x00000151, 0x0000030B},+{0x00000154, 0x00000052}, {0x00000154, 0x00000301}, {0x00000155, 0x00000072}, {0x00000155, 0x00000301},+{0x00000156, 0x00000052}, {0x00000156, 0x00000327}, {0x00000157, 0x00000072}, {0x00000157, 0x00000327},+{0x00000158, 0x00000052}, {0x00000158, 0x0000030C}, {0x00000159, 0x00000072}, {0x00000159, 0x0000030C},+{0x0000015A, 0x00000053}, {0x0000015A, 0x00000301}, {0x0000015B, 0x00000073}, {0x0000015B, 0x00000301},+{0x0000015C, 0x00000053}, {0x0000015C, 0x00000302}, {0x0000015D, 0x00000073}, {0x0000015D, 0x00000302},+{0x0000015E, 0x00000053}, {0x0000015E, 0x00000327}, {0x0000015F, 0x00000073}, {0x0000015F, 0x00000327},+{0x00000160, 0x00000053}, {0x00000160, 0x0000030C}, {0x00000161, 0x00000073}, {0x00000161, 0x0000030C},+{0x00000162, 0x00000054}, {0x00000162, 0x00000327}, {0x00000163, 0x00000074}, {0x00000163, 0x00000327},+{0x00000164, 0x00000054}, {0x00000164, 0x0000030C}, {0x00000165, 0x00000074}, {0x00000165, 0x0000030C},+{0x00000168, 0x00000055}, {0x00000168, 0x00000303}, {0x00000169, 0x00000075}, {0x00000169, 0x00000303},+{0x0000016A, 0x00000055}, {0x0000016A, 0x00000304}, {0x0000016B, 0x00000075}, {0x0000016B, 0x00000304},+{0x0000016C, 0x00000055}, {0x0000016C, 0x00000306}, {0x0000016D, 0x00000075}, {0x0000016D, 0x00000306},+{0x0000016E, 0x00000055}, {0x0000016E, 0x0000030A}, {0x0000016F, 0x00000075}, {0x0000016F, 0x0000030A},+{0x00000170, 0x00000055}, {0x00000170, 0x0000030B}, {0x00000171, 0x00000075}, {0x00000171, 0x0000030B},+{0x00000172, 0x00000055}, {0x00000172, 0x00000328}, {0x00000173, 0x00000075}, {0x00000173, 0x00000328},+{0x00000174, 0x00000057}, {0x00000174, 0x00000302}, {0x00000175, 0x00000077}, {0x00000175, 0x00000302},+{0x00000176, 0x00000059}, {0x00000176, 0x00000302}, {0x00000177, 0x00000079}, {0x00000177, 0x00000302},+{0x00000178, 0x00000059}, {0x00000178, 0x00000308}, {0x00000179, 0x0000005A}, {0x00000179, 0x00000301},+{0x0000017A, 0x0000007A}, {0x0000017A, 0x00000301}, {0x0000017B, 0x0000005A}, {0x0000017B, 0x00000307},+{0x0000017C, 0x0000007A}, {0x0000017C, 0x00000307}, {0x0000017D, 0x0000005A}, {0x0000017D, 0x0000030C},+{0x0000017E, 0x0000007A}, {0x0000017E, 0x0000030C}, {0x000001A0, 0x0000004F}, {0x000001A0, 0x0000031B},+{0x000001A1, 0x0000006F}, {0x000001A1, 0x0000031B}, {0x000001AF, 0x00000055}, {0x000001AF, 0x0000031B},+{0x000001B0, 0x00000075}, {0x000001B0, 0x0000031B}, {0x000001CD, 0x00000041}, {0x000001CD, 0x0000030C},+{0x000001CE, 0x00000061}, {0x000001CE, 0x0000030C}, {0x000001CF, 0x00000049}, {0x000001CF, 0x0000030C},+{0x000001D0, 0x00000069}, {0x000001D0, 0x0000030C}, {0x000001D1, 0x0000004F}, {0x000001D1, 0x0000030C},+{0x000001D2, 0x0000006F}, {0x000001D2, 0x0000030C}, {0x000001D3, 0x00000055}, {0x000001D3, 0x0000030C},+{0x000001D4, 0x00000075}, {0x000001D4, 0x0000030C}, {0x000001D5, 0x00000055}, {0x000001D5, 0x00000308},+{0x000001D5, 0x00000304}, {0x000001D6, 0x00000075}, {0x000001D6, 0x00000308}, {0x000001D6, 0x00000304},+{0x000001D7, 0x00000055}, {0x000001D7, 0x00000308}, {0x000001D7, 0x00000301}, {0x000001D8, 0x00000075},+{0x000001D8, 0x00000308}, {0x000001D8, 0x00000301}, {0x000001D9, 0x00000055}, {0x000001D9, 0x00000308},+{0x000001D9, 0x0000030C}, {0x000001DA, 0x00000075}, {0x000001DA, 0x00000308}, {0x000001DA, 0x0000030C},+{0x000001DB, 0x00000055}, {0x000001DB, 0x00000308}, {0x000001DB, 0x00000300}, {0x000001DC, 0x00000075},+{0x000001DC, 0x00000308}, {0x000001DC, 0x00000300}, {0x000001DE, 0x00000041}, {0x000001DE, 0x00000308},+{0x000001DE, 0x00000304}, {0x000001DF, 0x00000061}, {0x000001DF, 0x00000308}, {0x000001DF, 0x00000304},+{0x000001E0, 0x00000041}, {0x000001E0, 0x00000307}, {0x000001E0, 0x00000304}, {0x000001E1, 0x00000061},+{0x000001E1, 0x00000307}, {0x000001E1, 0x00000304}, {0x000001E2, 0x000000C6}, {0x000001E2, 0x00000304},+{0x000001E3, 0x000000E6}, {0x000001E3, 0x00000304}, {0x000001E6, 0x00000047}, {0x000001E6, 0x0000030C},+{0x000001E7, 0x00000067}, {0x000001E7, 0x0000030C}, {0x000001E8, 0x0000004B}, {0x000001E8, 0x0000030C},+{0x000001E9, 0x0000006B}, {0x000001E9, 0x0000030C}, {0x000001EA, 0x0000004F}, {0x000001EA, 0x00000328},+{0x000001EB, 0x0000006F}, {0x000001EB, 0x00000328}, {0x000001EC, 0x0000004F}, {0x000001EC, 0x00000328},+{0x000001EC, 0x00000304}, {0x000001ED, 0x0000006F}, {0x000001ED, 0x00000328}, {0x000001ED, 0x00000304},+{0x000001EE, 0x000001B7}, {0x000001EE, 0x0000030C}, {0x000001EF, 0x00000292}, {0x000001EF, 0x0000030C},+{0x000001F0, 0x0000006A}, {0x000001F0, 0x0000030C}, {0x000001F4, 0x00000047}, {0x000001F4, 0x00000301},+{0x000001F5, 0x00000067}, {0x000001F5, 0x00000301}, {0x000001F8, 0x0000004E}, {0x000001F8, 0x00000300},+{0x000001F9, 0x0000006E}, {0x000001F9, 0x00000300}, {0x000001FA, 0x00000041}, {0x000001FA, 0x0000030A},+{0x000001FA, 0x00000301}, {0x000001FB, 0x00000061}, {0x000001FB, 0x0000030A}, {0x000001FB, 0x00000301},+{0x000001FC, 0x000000C6}, {0x000001FC, 0x00000301}, {0x000001FD, 0x000000E6}, {0x000001FD, 0x00000301},+{0x000001FE, 0x000000D8}, {0x000001FE, 0x00000301}, {0x000001FF, 0x000000F8}, {0x000001FF, 0x00000301},+{0x00000200, 0x00000041}, {0x00000200, 0x0000030F}, {0x00000201, 0x00000061}, {0x00000201, 0x0000030F},+{0x00000202, 0x00000041}, {0x00000202, 0x00000311}, {0x00000203, 0x00000061}, {0x00000203, 0x00000311},+{0x00000204, 0x00000045}, {0x00000204, 0x0000030F}, {0x00000205, 0x00000065}, {0x00000205, 0x0000030F},+{0x00000206, 0x00000045}, {0x00000206, 0x00000311}, {0x00000207, 0x00000065}, {0x00000207, 0x00000311},+{0x00000208, 0x00000049}, {0x00000208, 0x0000030F}, {0x00000209, 0x00000069}, {0x00000209, 0x0000030F},+{0x0000020A, 0x00000049}, {0x0000020A, 0x00000311}, {0x0000020B, 0x00000069}, {0x0000020B, 0x00000311},+{0x0000020C, 0x0000004F}, {0x0000020C, 0x0000030F}, {0x0000020D, 0x0000006F}, {0x0000020D, 0x0000030F},+{0x0000020E, 0x0000004F}, {0x0000020E, 0x00000311}, {0x0000020F, 0x0000006F}, {0x0000020F, 0x00000311},+{0x00000210, 0x00000052}, {0x00000210, 0x0000030F}, {0x00000211, 0x00000072}, {0x00000211, 0x0000030F},+{0x00000212, 0x00000052}, {0x00000212, 0x00000311}, {0x00000213, 0x00000072}, {0x00000213, 0x00000311},+{0x00000214, 0x00000055}, {0x00000214, 0x0000030F}, {0x00000215, 0x00000075}, {0x00000215, 0x0000030F},+{0x00000216, 0x00000055}, {0x00000216, 0x00000311}, {0x00000217, 0x00000075}, {0x00000217, 0x00000311},+{0x00000218, 0x00000053}, {0x00000218, 0x00000326}, {0x00000219, 0x00000073}, {0x00000219, 0x00000326},+{0x0000021A, 0x00000054}, {0x0000021A, 0x00000326}, {0x0000021B, 0x00000074}, {0x0000021B, 0x00000326},+{0x0000021E, 0x00000048}, {0x0000021E, 0x0000030C}, {0x0000021F, 0x00000068}, {0x0000021F, 0x0000030C},+{0x00000226, 0x00000041}, {0x00000226, 0x00000307}, {0x00000227, 0x00000061}, {0x00000227, 0x00000307},+{0x00000228, 0x00000045}, {0x00000228, 0x00000327}, {0x00000229, 0x00000065}, {0x00000229, 0x00000327},+{0x0000022A, 0x0000004F}, {0x0000022A, 0x00000308}, {0x0000022A, 0x00000304}, {0x0000022B, 0x0000006F},+{0x0000022B, 0x00000308}, {0x0000022B, 0x00000304}, {0x0000022C, 0x0000004F}, {0x0000022C, 0x00000303},+{0x0000022C, 0x00000304}, {0x0000022D, 0x0000006F}, {0x0000022D, 0x00000303}, {0x0000022D, 0x00000304},+{0x0000022E, 0x0000004F}, {0x0000022E, 0x00000307}, {0x0000022F, 0x0000006F}, {0x0000022F, 0x00000307},+{0x00000230, 0x0000004F}, {0x00000230, 0x00000307}, {0x00000230, 0x00000304}, {0x00000231, 0x0000006F},+{0x00000231, 0x00000307}, {0x00000231, 0x00000304}, {0x00000232, 0x00000059}, {0x00000232, 0x00000304},+{0x00000233, 0x00000079}, {0x00000233, 0x00000304}, {0x00000340, 0x00000300}, {0x00000341, 0x00000301},+{0x00000343, 0x00000313}, {0x00000344, 0x00000308}, {0x00000344, 0x00000301}, {0x00000374, 0x000002B9},+{0x0000037E, 0x0000003B}, {0x00000385, 0x000000A8}, {0x00000385, 0x00000301}, {0x00000386, 0x00000391},+{0x00000386, 0x00000301}, {0x00000387, 0x000000B7}, {0x00000388, 0x00000395}, {0x00000388, 0x00000301},+{0x00000389, 0x00000397}, {0x00000389, 0x00000301}, {0x0000038A, 0x00000399}, {0x0000038A, 0x00000301},+{0x0000038C, 0x0000039F}, {0x0000038C, 0x00000301}, {0x0000038E, 0x000003A5}, {0x0000038E, 0x00000301},+{0x0000038F, 0x000003A9}, {0x0000038F, 0x00000301}, {0x00000390, 0x000003B9}, {0x00000390, 0x00000308},+{0x00000390, 0x00000301}, {0x000003AA, 0x00000399}, {0x000003AA, 0x00000308}, {0x000003AB, 0x000003A5},+{0x000003AB, 0x00000308}, {0x000003AC, 0x000003B1}, {0x000003AC, 0x00000301}, {0x000003AD, 0x000003B5},+{0x000003AD, 0x00000301}, {0x000003AE, 0x000003B7}, {0x000003AE, 0x00000301}, {0x000003AF, 0x000003B9},+{0x000003AF, 0x00000301}, {0x000003B0, 0x000003C5}, {0x000003B0, 0x00000308}, {0x000003B0, 0x00000301},+{0x000003CA, 0x000003B9}, {0x000003CA, 0x00000308}, {0x000003CB, 0x000003C5}, {0x000003CB, 0x00000308},+{0x000003CC, 0x000003BF}, {0x000003CC, 0x00000301}, {0x000003CD, 0x000003C5}, {0x000003CD, 0x00000301},+{0x000003CE, 0x000003C9}, {0x000003CE, 0x00000301}, {0x000003D3, 0x000003D2}, {0x000003D3, 0x00000301},+{0x000003D4, 0x000003D2}, {0x000003D4, 0x00000308}, {0x00000400, 0x00000415}, {0x00000400, 0x00000300},+{0x00000401, 0x00000415}, {0x00000401, 0x00000308}, {0x00000403, 0x00000413}, {0x00000403, 0x00000301},+{0x00000407, 0x00000406}, {0x00000407, 0x00000308}, {0x0000040C, 0x0000041A}, {0x0000040C, 0x00000301},+{0x0000040D, 0x00000418}, {0x0000040D, 0x00000300}, {0x0000040E, 0x00000423}, {0x0000040E, 0x00000306},+{0x00000419, 0x00000418}, {0x00000419, 0x00000306}, {0x00000439, 0x00000438}, {0x00000439, 0x00000306},+{0x00000450, 0x00000435}, {0x00000450, 0x00000300}, {0x00000451, 0x00000435}, {0x00000451, 0x00000308},+{0x00000453, 0x00000433}, {0x00000453, 0x00000301}, {0x00000457, 0x00000456}, {0x00000457, 0x00000308},+{0x0000045C, 0x0000043A}, {0x0000045C, 0x00000301}, {0x0000045D, 0x00000438}, {0x0000045D, 0x00000300},+{0x0000045E, 0x00000443}, {0x0000045E, 0x00000306}, {0x00000476, 0x00000474}, {0x00000476, 0x0000030F},+{0x00000477, 0x00000475}, {0x00000477, 0x0000030F}, {0x000004C1, 0x00000416}, {0x000004C1, 0x00000306},+{0x000004C2, 0x00000436}, {0x000004C2, 0x00000306}, {0x000004D0, 0x00000410}, {0x000004D0, 0x00000306},+{0x000004D1, 0x00000430}, {0x000004D1, 0x00000306}, {0x000004D2, 0x00000410}, {0x000004D2, 0x00000308},+{0x000004D3, 0x00000430}, {0x000004D3, 0x00000308}, {0x000004D6, 0x00000415}, {0x000004D6, 0x00000306},+{0x000004D7, 0x00000435}, {0x000004D7, 0x00000306}, {0x000004DA, 0x000004D8}, {0x000004DA, 0x00000308},+{0x000004DB, 0x000004D9}, {0x000004DB, 0x00000308}, {0x000004DC, 0x00000416}, {0x000004DC, 0x00000308},+{0x000004DD, 0x00000436}, {0x000004DD, 0x00000308}, {0x000004DE, 0x00000417}, {0x000004DE, 0x00000308},+{0x000004DF, 0x00000437}, {0x000004DF, 0x00000308}, {0x000004E2, 0x00000418}, {0x000004E2, 0x00000304},+{0x000004E3, 0x00000438}, {0x000004E3, 0x00000304}, {0x000004E4, 0x00000418}, {0x000004E4, 0x00000308},+{0x000004E5, 0x00000438}, {0x000004E5, 0x00000308}, {0x000004E6, 0x0000041E}, {0x000004E6, 0x00000308},+{0x000004E7, 0x0000043E}, {0x000004E7, 0x00000308}, {0x000004EA, 0x000004E8}, {0x000004EA, 0x00000308},+{0x000004EB, 0x000004E9}, {0x000004EB, 0x00000308}, {0x000004EC, 0x0000042D}, {0x000004EC, 0x00000308},+{0x000004ED, 0x0000044D}, {0x000004ED, 0x00000308}, {0x000004EE, 0x00000423}, {0x000004EE, 0x00000304},+{0x000004EF, 0x00000443}, {0x000004EF, 0x00000304}, {0x000004F0, 0x00000423}, {0x000004F0, 0x00000308},+{0x000004F1, 0x00000443}, {0x000004F1, 0x00000308}, {0x000004F2, 0x00000423}, {0x000004F2, 0x0000030B},+{0x000004F3, 0x00000443}, {0x000004F3, 0x0000030B}, {0x000004F4, 0x00000427}, {0x000004F4, 0x00000308},+{0x000004F5, 0x00000447}, {0x000004F5, 0x00000308}, {0x000004F8, 0x0000042B}, {0x000004F8, 0x00000308},+{0x000004F9, 0x0000044B}, {0x000004F9, 0x00000308}, {0x00000622, 0x00000627}, {0x00000622, 0x00000653},+{0x00000623, 0x00000627}, {0x00000623, 0x00000654}, {0x00000624, 0x00000648}, {0x00000624, 0x00000654},+{0x00000625, 0x00000627}, {0x00000625, 0x00000655}, {0x00000626, 0x0000064A}, {0x00000626, 0x00000654},+{0x000006C0, 0x000006D5}, {0x000006C0, 0x00000654}, {0x000006C2, 0x000006C1}, {0x000006C2, 0x00000654},+{0x000006D3, 0x000006D2}, {0x000006D3, 0x00000654}, {0x00000929, 0x00000928}, {0x00000929, 0x0000093C},+{0x00000931, 0x00000930}, {0x00000931, 0x0000093C}, {0x00000934, 0x00000933}, {0x00000934, 0x0000093C},+{0x00000958, 0x00000915}, {0x00000958, 0x0000093C}, {0x00000959, 0x00000916}, {0x00000959, 0x0000093C},+{0x0000095A, 0x00000917}, {0x0000095A, 0x0000093C}, {0x0000095B, 0x0000091C}, {0x0000095B, 0x0000093C},+{0x0000095C, 0x00000921}, {0x0000095C, 0x0000093C}, {0x0000095D, 0x00000922}, {0x0000095D, 0x0000093C},+{0x0000095E, 0x0000092B}, {0x0000095E, 0x0000093C}, {0x0000095F, 0x0000092F}, {0x0000095F, 0x0000093C},+{0x000009CB, 0x000009C7}, {0x000009CB, 0x000009BE}, {0x000009CC, 0x000009C7}, {0x000009CC, 0x000009D7},+{0x000009DC, 0x000009A1}, {0x000009DC, 0x000009BC}, {0x000009DD, 0x000009A2}, {0x000009DD, 0x000009BC},+{0x000009DF, 0x000009AF}, {0x000009DF, 0x000009BC}, {0x00000A33, 0x00000A32}, {0x00000A33, 0x00000A3C},+{0x00000A36, 0x00000A38}, {0x00000A36, 0x00000A3C}, {0x00000A59, 0x00000A16}, {0x00000A59, 0x00000A3C},+{0x00000A5A, 0x00000A17}, {0x00000A5A, 0x00000A3C}, {0x00000A5B, 0x00000A1C}, {0x00000A5B, 0x00000A3C},+{0x00000A5E, 0x00000A2B}, {0x00000A5E, 0x00000A3C}, {0x00000B48, 0x00000B47}, {0x00000B48, 0x00000B56},+{0x00000B4B, 0x00000B47}, {0x00000B4B, 0x00000B3E}, {0x00000B4C, 0x00000B47}, {0x00000B4C, 0x00000B57},+{0x00000B5C, 0x00000B21}, {0x00000B5C, 0x00000B3C}, {0x00000B5D, 0x00000B22}, {0x00000B5D, 0x00000B3C},+{0x00000B94, 0x00000B92}, {0x00000B94, 0x00000BD7}, {0x00000BCA, 0x00000BC6}, {0x00000BCA, 0x00000BBE},+{0x00000BCB, 0x00000BC7}, {0x00000BCB, 0x00000BBE}, {0x00000BCC, 0x00000BC6}, {0x00000BCC, 0x00000BD7},+{0x00000C48, 0x00000C46}, {0x00000C48, 0x00000C56}, {0x00000CC0, 0x00000CBF}, {0x00000CC0, 0x00000CD5},+{0x00000CC7, 0x00000CC6}, {0x00000CC7, 0x00000CD5}, {0x00000CC8, 0x00000CC6}, {0x00000CC8, 0x00000CD6},+{0x00000CCA, 0x00000CC6}, {0x00000CCA, 0x00000CC2}, {0x00000CCB, 0x00000CC6}, {0x00000CCB, 0x00000CC2},+{0x00000CCB, 0x00000CD5}, {0x00000D4A, 0x00000D46}, {0x00000D4A, 0x00000D3E}, {0x00000D4B, 0x00000D47},+{0x00000D4B, 0x00000D3E}, {0x00000D4C, 0x00000D46}, {0x00000D4C, 0x00000D57}, {0x00000DDA, 0x00000DD9},+{0x00000DDA, 0x00000DCA}, {0x00000DDC, 0x00000DD9}, {0x00000DDC, 0x00000DCF}, {0x00000DDD, 0x00000DD9},+{0x00000DDD, 0x00000DCF}, {0x00000DDD, 0x00000DCA}, {0x00000DDE, 0x00000DD9}, {0x00000DDE, 0x00000DDF},+{0x00000F43, 0x00000F42}, {0x00000F43, 0x00000FB7}, {0x00000F4D, 0x00000F4C}, {0x00000F4D, 0x00000FB7},+{0x00000F52, 0x00000F51}, {0x00000F52, 0x00000FB7}, {0x00000F57, 0x00000F56}, {0x00000F57, 0x00000FB7},+{0x00000F5C, 0x00000F5B}, {0x00000F5C, 0x00000FB7}, {0x00000F69, 0x00000F40}, {0x00000F69, 0x00000FB5},+{0x00000F73, 0x00000F71}, {0x00000F73, 0x00000F72}, {0x00000F75, 0x00000F71}, {0x00000F75, 0x00000F74},+{0x00000F76, 0x00000FB2}, {0x00000F76, 0x00000F80}, {0x00000F78, 0x00000FB3}, {0x00000F78, 0x00000F80},+{0x00000F81, 0x00000F71}, {0x00000F81, 0x00000F80}, {0x00000F93, 0x00000F92}, {0x00000F93, 0x00000FB7},+{0x00000F9D, 0x00000F9C}, {0x00000F9D, 0x00000FB7}, {0x00000FA2, 0x00000FA1}, {0x00000FA2, 0x00000FB7},+{0x00000FA7, 0x00000FA6}, {0x00000FA7, 0x00000FB7}, {0x00000FAC, 0x00000FAB}, {0x00000FAC, 0x00000FB7},+{0x00000FB9, 0x00000F90}, {0x00000FB9, 0x00000FB5}, {0x00001026, 0x00001025}, {0x00001026, 0x0000102E},+{0x00001B06, 0x00001B05}, {0x00001B06, 0x00001B35}, {0x00001B08, 0x00001B07}, {0x00001B08, 0x00001B35},+{0x00001B0A, 0x00001B09}, {0x00001B0A, 0x00001B35}, {0x00001B0C, 0x00001B0B}, {0x00001B0C, 0x00001B35},+{0x00001B0E, 0x00001B0D}, {0x00001B0E, 0x00001B35}, {0x00001B12, 0x00001B11}, {0x00001B12, 0x00001B35},+{0x00001B3B, 0x00001B3A}, {0x00001B3B, 0x00001B35}, {0x00001B3D, 0x00001B3C}, {0x00001B3D, 0x00001B35},+{0x00001B40, 0x00001B3E}, {0x00001B40, 0x00001B35}, {0x00001B41, 0x00001B3F}, {0x00001B41, 0x00001B35},+{0x00001B43, 0x00001B42}, {0x00001B43, 0x00001B35}, {0x00001E00, 0x00000041}, {0x00001E00, 0x00000325},+{0x00001E01, 0x00000061}, {0x00001E01, 0x00000325}, {0x00001E02, 0x00000042}, {0x00001E02, 0x00000307},+{0x00001E03, 0x00000062}, {0x00001E03, 0x00000307}, {0x00001E04, 0x00000042}, {0x00001E04, 0x00000323},+{0x00001E05, 0x00000062}, {0x00001E05, 0x00000323}, {0x00001E06, 0x00000042}, {0x00001E06, 0x00000331},+{0x00001E07, 0x00000062}, {0x00001E07, 0x00000331}, {0x00001E08, 0x00000043}, {0x00001E08, 0x00000327},+{0x00001E08, 0x00000301}, {0x00001E09, 0x00000063}, {0x00001E09, 0x00000327}, {0x00001E09, 0x00000301},+{0x00001E0A, 0x00000044}, {0x00001E0A, 0x00000307}, {0x00001E0B, 0x00000064}, {0x00001E0B, 0x00000307},+{0x00001E0C, 0x00000044}, {0x00001E0C, 0x00000323}, {0x00001E0D, 0x00000064}, {0x00001E0D, 0x00000323},+{0x00001E0E, 0x00000044}, {0x00001E0E, 0x00000331}, {0x00001E0F, 0x00000064}, {0x00001E0F, 0x00000331},+{0x00001E10, 0x00000044}, {0x00001E10, 0x00000327}, {0x00001E11, 0x00000064}, {0x00001E11, 0x00000327},+{0x00001E12, 0x00000044}, {0x00001E12, 0x0000032D}, {0x00001E13, 0x00000064}, {0x00001E13, 0x0000032D},+{0x00001E14, 0x00000045}, {0x00001E14, 0x00000304}, {0x00001E14, 0x00000300}, {0x00001E15, 0x00000065},+{0x00001E15, 0x00000304}, {0x00001E15, 0x00000300}, {0x00001E16, 0x00000045}, {0x00001E16, 0x00000304},+{0x00001E16, 0x00000301}, {0x00001E17, 0x00000065}, {0x00001E17, 0x00000304}, {0x00001E17, 0x00000301},+{0x00001E18, 0x00000045}, {0x00001E18, 0x0000032D}, {0x00001E19, 0x00000065}, {0x00001E19, 0x0000032D},+{0x00001E1A, 0x00000045}, {0x00001E1A, 0x00000330}, {0x00001E1B, 0x00000065}, {0x00001E1B, 0x00000330},+{0x00001E1C, 0x00000045}, {0x00001E1C, 0x00000327}, {0x00001E1C, 0x00000306}, {0x00001E1D, 0x00000065},+{0x00001E1D, 0x00000327}, {0x00001E1D, 0x00000306}, {0x00001E1E, 0x00000046}, {0x00001E1E, 0x00000307},+{0x00001E1F, 0x00000066}, {0x00001E1F, 0x00000307}, {0x00001E20, 0x00000047}, {0x00001E20, 0x00000304},+{0x00001E21, 0x00000067}, {0x00001E21, 0x00000304}, {0x00001E22, 0x00000048}, {0x00001E22, 0x00000307},+{0x00001E23, 0x00000068}, {0x00001E23, 0x00000307}, {0x00001E24, 0x00000048}, {0x00001E24, 0x00000323},+{0x00001E25, 0x00000068}, {0x00001E25, 0x00000323}, {0x00001E26, 0x00000048}, {0x00001E26, 0x00000308},+{0x00001E27, 0x00000068}, {0x00001E27, 0x00000308}, {0x00001E28, 0x00000048}, {0x00001E28, 0x00000327},+{0x00001E29, 0x00000068}, {0x00001E29, 0x00000327}, {0x00001E2A, 0x00000048}, {0x00001E2A, 0x0000032E},+{0x00001E2B, 0x00000068}, {0x00001E2B, 0x0000032E}, {0x00001E2C, 0x00000049}, {0x00001E2C, 0x00000330},+{0x00001E2D, 0x00000069}, {0x00001E2D, 0x00000330}, {0x00001E2E, 0x00000049}, {0x00001E2E, 0x00000308},+{0x00001E2E, 0x00000301}, {0x00001E2F, 0x00000069}, {0x00001E2F, 0x00000308}, {0x00001E2F, 0x00000301},+{0x00001E30, 0x0000004B}, {0x00001E30, 0x00000301}, {0x00001E31, 0x0000006B}, {0x00001E31, 0x00000301},+{0x00001E32, 0x0000004B}, {0x00001E32, 0x00000323}, {0x00001E33, 0x0000006B}, {0x00001E33, 0x00000323},+{0x00001E34, 0x0000004B}, {0x00001E34, 0x00000331}, {0x00001E35, 0x0000006B}, {0x00001E35, 0x00000331},+{0x00001E36, 0x0000004C}, {0x00001E36, 0x00000323}, {0x00001E37, 0x0000006C}, {0x00001E37, 0x00000323},+{0x00001E38, 0x0000004C}, {0x00001E38, 0x00000323}, {0x00001E38, 0x00000304}, {0x00001E39, 0x0000006C},+{0x00001E39, 0x00000323}, {0x00001E39, 0x00000304}, {0x00001E3A, 0x0000004C}, {0x00001E3A, 0x00000331},+{0x00001E3B, 0x0000006C}, {0x00001E3B, 0x00000331}, {0x00001E3C, 0x0000004C}, {0x00001E3C, 0x0000032D},+{0x00001E3D, 0x0000006C}, {0x00001E3D, 0x0000032D}, {0x00001E3E, 0x0000004D}, {0x00001E3E, 0x00000301},+{0x00001E3F, 0x0000006D}, {0x00001E3F, 0x00000301}, {0x00001E40, 0x0000004D}, {0x00001E40, 0x00000307},+{0x00001E41, 0x0000006D}, {0x00001E41, 0x00000307}, {0x00001E42, 0x0000004D}, {0x00001E42, 0x00000323},+{0x00001E43, 0x0000006D}, {0x00001E43, 0x00000323}, {0x00001E44, 0x0000004E}, {0x00001E44, 0x00000307},+{0x00001E45, 0x0000006E}, {0x00001E45, 0x00000307}, {0x00001E46, 0x0000004E}, {0x00001E46, 0x00000323},+{0x00001E47, 0x0000006E}, {0x00001E47, 0x00000323}, {0x00001E48, 0x0000004E}, {0x00001E48, 0x00000331},+{0x00001E49, 0x0000006E}, {0x00001E49, 0x00000331}, {0x00001E4A, 0x0000004E}, {0x00001E4A, 0x0000032D},+{0x00001E4B, 0x0000006E}, {0x00001E4B, 0x0000032D}, {0x00001E4C, 0x0000004F}, {0x00001E4C, 0x00000303},+{0x00001E4C, 0x00000301}, {0x00001E4D, 0x0000006F}, {0x00001E4D, 0x00000303}, {0x00001E4D, 0x00000301},+{0x00001E4E, 0x0000004F}, {0x00001E4E, 0x00000303}, {0x00001E4E, 0x00000308}, {0x00001E4F, 0x0000006F},+{0x00001E4F, 0x00000303}, {0x00001E4F, 0x00000308}, {0x00001E50, 0x0000004F}, {0x00001E50, 0x00000304},+{0x00001E50, 0x00000300}, {0x00001E51, 0x0000006F}, {0x00001E51, 0x00000304}, {0x00001E51, 0x00000300},+{0x00001E52, 0x0000004F}, {0x00001E52, 0x00000304}, {0x00001E52, 0x00000301}, {0x00001E53, 0x0000006F},+{0x00001E53, 0x00000304}, {0x00001E53, 0x00000301}, {0x00001E54, 0x00000050}, {0x00001E54, 0x00000301},+{0x00001E55, 0x00000070}, {0x00001E55, 0x00000301}, {0x00001E56, 0x00000050}, {0x00001E56, 0x00000307},+{0x00001E57, 0x00000070}, {0x00001E57, 0x00000307}, {0x00001E58, 0x00000052}, {0x00001E58, 0x00000307},+{0x00001E59, 0x00000072}, {0x00001E59, 0x00000307}, {0x00001E5A, 0x00000052}, {0x00001E5A, 0x00000323},+{0x00001E5B, 0x00000072}, {0x00001E5B, 0x00000323}, {0x00001E5C, 0x00000052}, {0x00001E5C, 0x00000323},+{0x00001E5C, 0x00000304}, {0x00001E5D, 0x00000072}, {0x00001E5D, 0x00000323}, {0x00001E5D, 0x00000304},+{0x00001E5E, 0x00000052}, {0x00001E5E, 0x00000331}, {0x00001E5F, 0x00000072}, {0x00001E5F, 0x00000331},+{0x00001E60, 0x00000053}, {0x00001E60, 0x00000307}, {0x00001E61, 0x00000073}, {0x00001E61, 0x00000307},+{0x00001E62, 0x00000053}, {0x00001E62, 0x00000323}, {0x00001E63, 0x00000073}, {0x00001E63, 0x00000323},+{0x00001E64, 0x00000053}, {0x00001E64, 0x00000301}, {0x00001E64, 0x00000307}, {0x00001E65, 0x00000073},+{0x00001E65, 0x00000301}, {0x00001E65, 0x00000307}, {0x00001E66, 0x00000053}, {0x00001E66, 0x0000030C},+{0x00001E66, 0x00000307}, {0x00001E67, 0x00000073}, {0x00001E67, 0x0000030C}, {0x00001E67, 0x00000307},+{0x00001E68, 0x00000053}, {0x00001E68, 0x00000323}, {0x00001E68, 0x00000307}, {0x00001E69, 0x00000073},+{0x00001E69, 0x00000323}, {0x00001E69, 0x00000307}, {0x00001E6A, 0x00000054}, {0x00001E6A, 0x00000307},+{0x00001E6B, 0x00000074}, {0x00001E6B, 0x00000307}, {0x00001E6C, 0x00000054}, {0x00001E6C, 0x00000323},+{0x00001E6D, 0x00000074}, {0x00001E6D, 0x00000323}, {0x00001E6E, 0x00000054}, {0x00001E6E, 0x00000331},+{0x00001E6F, 0x00000074}, {0x00001E6F, 0x00000331}, {0x00001E70, 0x00000054}, {0x00001E70, 0x0000032D},+{0x00001E71, 0x00000074}, {0x00001E71, 0x0000032D}, {0x00001E72, 0x00000055}, {0x00001E72, 0x00000324},+{0x00001E73, 0x00000075}, {0x00001E73, 0x00000324}, {0x00001E74, 0x00000055}, {0x00001E74, 0x00000330},+{0x00001E75, 0x00000075}, {0x00001E75, 0x00000330}, {0x00001E76, 0x00000055}, {0x00001E76, 0x0000032D},+{0x00001E77, 0x00000075}, {0x00001E77, 0x0000032D}, {0x00001E78, 0x00000055}, {0x00001E78, 0x00000303},+{0x00001E78, 0x00000301}, {0x00001E79, 0x00000075}, {0x00001E79, 0x00000303}, {0x00001E79, 0x00000301},+{0x00001E7A, 0x00000055}, {0x00001E7A, 0x00000304}, {0x00001E7A, 0x00000308}, {0x00001E7B, 0x00000075},+{0x00001E7B, 0x00000304}, {0x00001E7B, 0x00000308}, {0x00001E7C, 0x00000056}, {0x00001E7C, 0x00000303},+{0x00001E7D, 0x00000076}, {0x00001E7D, 0x00000303}, {0x00001E7E, 0x00000056}, {0x00001E7E, 0x00000323},+{0x00001E7F, 0x00000076}, {0x00001E7F, 0x00000323}, {0x00001E80, 0x00000057}, {0x00001E80, 0x00000300},+{0x00001E81, 0x00000077}, {0x00001E81, 0x00000300}, {0x00001E82, 0x00000057}, {0x00001E82, 0x00000301},+{0x00001E83, 0x00000077}, {0x00001E83, 0x00000301}, {0x00001E84, 0x00000057}, {0x00001E84, 0x00000308},+{0x00001E85, 0x00000077}, {0x00001E85, 0x00000308}, {0x00001E86, 0x00000057}, {0x00001E86, 0x00000307},+{0x00001E87, 0x00000077}, {0x00001E87, 0x00000307}, {0x00001E88, 0x00000057}, {0x00001E88, 0x00000323},+{0x00001E89, 0x00000077}, {0x00001E89, 0x00000323}, {0x00001E8A, 0x00000058}, {0x00001E8A, 0x00000307},+{0x00001E8B, 0x00000078}, {0x00001E8B, 0x00000307}, {0x00001E8C, 0x00000058}, {0x00001E8C, 0x00000308},+{0x00001E8D, 0x00000078}, {0x00001E8D, 0x00000308}, {0x00001E8E, 0x00000059}, {0x00001E8E, 0x00000307},+{0x00001E8F, 0x00000079}, {0x00001E8F, 0x00000307}, {0x00001E90, 0x0000005A}, {0x00001E90, 0x00000302},+{0x00001E91, 0x0000007A}, {0x00001E91, 0x00000302}, {0x00001E92, 0x0000005A}, {0x00001E92, 0x00000323},+{0x00001E93, 0x0000007A}, {0x00001E93, 0x00000323}, {0x00001E94, 0x0000005A}, {0x00001E94, 0x00000331},+{0x00001E95, 0x0000007A}, {0x00001E95, 0x00000331}, {0x00001E96, 0x00000068}, {0x00001E96, 0x00000331},+{0x00001E97, 0x00000074}, {0x00001E97, 0x00000308}, {0x00001E98, 0x00000077}, {0x00001E98, 0x0000030A},+{0x00001E99, 0x00000079}, {0x00001E99, 0x0000030A}, {0x00001E9B, 0x0000017F}, {0x00001E9B, 0x00000307},+{0x00001EA0, 0x00000041}, {0x00001EA0, 0x00000323}, {0x00001EA1, 0x00000061}, {0x00001EA1, 0x00000323},+{0x00001EA2, 0x00000041}, {0x00001EA2, 0x00000309}, {0x00001EA3, 0x00000061}, {0x00001EA3, 0x00000309},+{0x00001EA4, 0x00000041}, {0x00001EA4, 0x00000302}, {0x00001EA4, 0x00000301}, {0x00001EA5, 0x00000061},+{0x00001EA5, 0x00000302}, {0x00001EA5, 0x00000301}, {0x00001EA6, 0x00000041}, {0x00001EA6, 0x00000302},+{0x00001EA6, 0x00000300}, {0x00001EA7, 0x00000061}, {0x00001EA7, 0x00000302}, {0x00001EA7, 0x00000300},+{0x00001EA8, 0x00000041}, {0x00001EA8, 0x00000302}, {0x00001EA8, 0x00000309}, {0x00001EA9, 0x00000061},+{0x00001EA9, 0x00000302}, {0x00001EA9, 0x00000309}, {0x00001EAA, 0x00000041}, {0x00001EAA, 0x00000302},+{0x00001EAA, 0x00000303}, {0x00001EAB, 0x00000061}, {0x00001EAB, 0x00000302}, {0x00001EAB, 0x00000303},+{0x00001EAC, 0x00000041}, {0x00001EAC, 0x00000323}, {0x00001EAC, 0x00000302}, {0x00001EAD, 0x00000061},+{0x00001EAD, 0x00000323}, {0x00001EAD, 0x00000302}, {0x00001EAE, 0x00000041}, {0x00001EAE, 0x00000306},+{0x00001EAE, 0x00000301}, {0x00001EAF, 0x00000061}, {0x00001EAF, 0x00000306}, {0x00001EAF, 0x00000301},+{0x00001EB0, 0x00000041}, {0x00001EB0, 0x00000306}, {0x00001EB0, 0x00000300}, {0x00001EB1, 0x00000061},+{0x00001EB1, 0x00000306}, {0x00001EB1, 0x00000300}, {0x00001EB2, 0x00000041}, {0x00001EB2, 0x00000306},+{0x00001EB2, 0x00000309}, {0x00001EB3, 0x00000061}, {0x00001EB3, 0x00000306}, {0x00001EB3, 0x00000309},+{0x00001EB4, 0x00000041}, {0x00001EB4, 0x00000306}, {0x00001EB4, 0x00000303}, {0x00001EB5, 0x00000061},+{0x00001EB5, 0x00000306}, {0x00001EB5, 0x00000303}, {0x00001EB6, 0x00000041}, {0x00001EB6, 0x00000323},+{0x00001EB6, 0x00000306}, {0x00001EB7, 0x00000061}, {0x00001EB7, 0x00000323}, {0x00001EB7, 0x00000306},+{0x00001EB8, 0x00000045}, {0x00001EB8, 0x00000323}, {0x00001EB9, 0x00000065}, {0x00001EB9, 0x00000323},+{0x00001EBA, 0x00000045}, {0x00001EBA, 0x00000309}, {0x00001EBB, 0x00000065}, {0x00001EBB, 0x00000309},+{0x00001EBC, 0x00000045}, {0x00001EBC, 0x00000303}, {0x00001EBD, 0x00000065}, {0x00001EBD, 0x00000303},+{0x00001EBE, 0x00000045}, {0x00001EBE, 0x00000302}, {0x00001EBE, 0x00000301}, {0x00001EBF, 0x00000065},+{0x00001EBF, 0x00000302}, {0x00001EBF, 0x00000301}, {0x00001EC0, 0x00000045}, {0x00001EC0, 0x00000302},+{0x00001EC0, 0x00000300}, {0x00001EC1, 0x00000065}, {0x00001EC1, 0x00000302}, {0x00001EC1, 0x00000300},+{0x00001EC2, 0x00000045}, {0x00001EC2, 0x00000302}, {0x00001EC2, 0x00000309}, {0x00001EC3, 0x00000065},+{0x00001EC3, 0x00000302}, {0x00001EC3, 0x00000309}, {0x00001EC4, 0x00000045}, {0x00001EC4, 0x00000302},+{0x00001EC4, 0x00000303}, {0x00001EC5, 0x00000065}, {0x00001EC5, 0x00000302}, {0x00001EC5, 0x00000303},+{0x00001EC6, 0x00000045}, {0x00001EC6, 0x00000323}, {0x00001EC6, 0x00000302}, {0x00001EC7, 0x00000065},+{0x00001EC7, 0x00000323}, {0x00001EC7, 0x00000302}, {0x00001EC8, 0x00000049}, {0x00001EC8, 0x00000309},+{0x00001EC9, 0x00000069}, {0x00001EC9, 0x00000309}, {0x00001ECA, 0x00000049}, {0x00001ECA, 0x00000323},+{0x00001ECB, 0x00000069}, {0x00001ECB, 0x00000323}, {0x00001ECC, 0x0000004F}, {0x00001ECC, 0x00000323},+{0x00001ECD, 0x0000006F}, {0x00001ECD, 0x00000323}, {0x00001ECE, 0x0000004F}, {0x00001ECE, 0x00000309},+{0x00001ECF, 0x0000006F}, {0x00001ECF, 0x00000309}, {0x00001ED0, 0x0000004F}, {0x00001ED0, 0x00000302},+{0x00001ED0, 0x00000301}, {0x00001ED1, 0x0000006F}, {0x00001ED1, 0x00000302}, {0x00001ED1, 0x00000301},+{0x00001ED2, 0x0000004F}, {0x00001ED2, 0x00000302}, {0x00001ED2, 0x00000300}, {0x00001ED3, 0x0000006F},+{0x00001ED3, 0x00000302}, {0x00001ED3, 0x00000300}, {0x00001ED4, 0x0000004F}, {0x00001ED4, 0x00000302},+{0x00001ED4, 0x00000309}, {0x00001ED5, 0x0000006F}, {0x00001ED5, 0x00000302}, {0x00001ED5, 0x00000309},+{0x00001ED6, 0x0000004F}, {0x00001ED6, 0x00000302}, {0x00001ED6, 0x00000303}, {0x00001ED7, 0x0000006F},+{0x00001ED7, 0x00000302}, {0x00001ED7, 0x00000303}, {0x00001ED8, 0x0000004F}, {0x00001ED8, 0x00000323},+{0x00001ED8, 0x00000302}, {0x00001ED9, 0x0000006F}, {0x00001ED9, 0x00000323}, {0x00001ED9, 0x00000302},+{0x00001EDA, 0x0000004F}, {0x00001EDA, 0x0000031B}, {0x00001EDA, 0x00000301}, {0x00001EDB, 0x0000006F},+{0x00001EDB, 0x0000031B}, {0x00001EDB, 0x00000301}, {0x00001EDC, 0x0000004F}, {0x00001EDC, 0x0000031B},+{0x00001EDC, 0x00000300}, {0x00001EDD, 0x0000006F}, {0x00001EDD, 0x0000031B}, {0x00001EDD, 0x00000300},+{0x00001EDE, 0x0000004F}, {0x00001EDE, 0x0000031B}, {0x00001EDE, 0x00000309}, {0x00001EDF, 0x0000006F},+{0x00001EDF, 0x0000031B}, {0x00001EDF, 0x00000309}, {0x00001EE0, 0x0000004F}, {0x00001EE0, 0x0000031B},+{0x00001EE0, 0x00000303}, {0x00001EE1, 0x0000006F}, {0x00001EE1, 0x0000031B}, {0x00001EE1, 0x00000303},+{0x00001EE2, 0x0000004F}, {0x00001EE2, 0x0000031B}, {0x00001EE2, 0x00000323}, {0x00001EE3, 0x0000006F},+{0x00001EE3, 0x0000031B}, {0x00001EE3, 0x00000323}, {0x00001EE4, 0x00000055}, {0x00001EE4, 0x00000323},+{0x00001EE5, 0x00000075}, {0x00001EE5, 0x00000323}, {0x00001EE6, 0x00000055}, {0x00001EE6, 0x00000309},+{0x00001EE7, 0x00000075}, {0x00001EE7, 0x00000309}, {0x00001EE8, 0x00000055}, {0x00001EE8, 0x0000031B},+{0x00001EE8, 0x00000301}, {0x00001EE9, 0x00000075}, {0x00001EE9, 0x0000031B}, {0x00001EE9, 0x00000301},+{0x00001EEA, 0x00000055}, {0x00001EEA, 0x0000031B}, {0x00001EEA, 0x00000300}, {0x00001EEB, 0x00000075},+{0x00001EEB, 0x0000031B}, {0x00001EEB, 0x00000300}, {0x00001EEC, 0x00000055}, {0x00001EEC, 0x0000031B},+{0x00001EEC, 0x00000309}, {0x00001EED, 0x00000075}, {0x00001EED, 0x0000031B}, {0x00001EED, 0x00000309},+{0x00001EEE, 0x00000055}, {0x00001EEE, 0x0000031B}, {0x00001EEE, 0x00000303}, {0x00001EEF, 0x00000075},+{0x00001EEF, 0x0000031B}, {0x00001EEF, 0x00000303}, {0x00001EF0, 0x00000055}, {0x00001EF0, 0x0000031B},+{0x00001EF0, 0x00000323}, {0x00001EF1, 0x00000075}, {0x00001EF1, 0x0000031B}, {0x00001EF1, 0x00000323},+{0x00001EF2, 0x00000059}, {0x00001EF2, 0x00000300}, {0x00001EF3, 0x00000079}, {0x00001EF3, 0x00000300},+{0x00001EF4, 0x00000059}, {0x00001EF4, 0x00000323}, {0x00001EF5, 0x00000079}, {0x00001EF5, 0x00000323},+{0x00001EF6, 0x00000059}, {0x00001EF6, 0x00000309}, {0x00001EF7, 0x00000079}, {0x00001EF7, 0x00000309},+{0x00001EF8, 0x00000059}, {0x00001EF8, 0x00000303}, {0x00001EF9, 0x00000079}, {0x00001EF9, 0x00000303},+{0x00001F00, 0x000003B1}, {0x00001F00, 0x00000313}, {0x00001F01, 0x000003B1}, {0x00001F01, 0x00000314},+{0x00001F02, 0x000003B1}, {0x00001F02, 0x00000313}, {0x00001F02, 0x00000300}, {0x00001F03, 0x000003B1},+{0x00001F03, 0x00000314}, {0x00001F03, 0x00000300}, {0x00001F04, 0x000003B1}, {0x00001F04, 0x00000313},+{0x00001F04, 0x00000301}, {0x00001F05, 0x000003B1}, {0x00001F05, 0x00000314}, {0x00001F05, 0x00000301},+{0x00001F06, 0x000003B1}, {0x00001F06, 0x00000313}, {0x00001F06, 0x00000342}, {0x00001F07, 0x000003B1},+{0x00001F07, 0x00000314}, {0x00001F07, 0x00000342}, {0x00001F08, 0x00000391}, {0x00001F08, 0x00000313},+{0x00001F09, 0x00000391}, {0x00001F09, 0x00000314}, {0x00001F0A, 0x00000391}, {0x00001F0A, 0x00000313},+{0x00001F0A, 0x00000300}, {0x00001F0B, 0x00000391}, {0x00001F0B, 0x00000314}, {0x00001F0B, 0x00000300},+{0x00001F0C, 0x00000391}, {0x00001F0C, 0x00000313}, {0x00001F0C, 0x00000301}, {0x00001F0D, 0x00000391},+{0x00001F0D, 0x00000314}, {0x00001F0D, 0x00000301}, {0x00001F0E, 0x00000391}, {0x00001F0E, 0x00000313},+{0x00001F0E, 0x00000342}, {0x00001F0F, 0x00000391}, {0x00001F0F, 0x00000314}, {0x00001F0F, 0x00000342},+{0x00001F10, 0x000003B5}, {0x00001F10, 0x00000313}, {0x00001F11, 0x000003B5}, {0x00001F11, 0x00000314},+{0x00001F12, 0x000003B5}, {0x00001F12, 0x00000313}, {0x00001F12, 0x00000300}, {0x00001F13, 0x000003B5},+{0x00001F13, 0x00000314}, {0x00001F13, 0x00000300}, {0x00001F14, 0x000003B5}, {0x00001F14, 0x00000313},+{0x00001F14, 0x00000301}, {0x00001F15, 0x000003B5}, {0x00001F15, 0x00000314}, {0x00001F15, 0x00000301},+{0x00001F18, 0x00000395}, {0x00001F18, 0x00000313}, {0x00001F19, 0x00000395}, {0x00001F19, 0x00000314},+{0x00001F1A, 0x00000395}, {0x00001F1A, 0x00000313}, {0x00001F1A, 0x00000300}, {0x00001F1B, 0x00000395},+{0x00001F1B, 0x00000314}, {0x00001F1B, 0x00000300}, {0x00001F1C, 0x00000395}, {0x00001F1C, 0x00000313},+{0x00001F1C, 0x00000301}, {0x00001F1D, 0x00000395}, {0x00001F1D, 0x00000314}, {0x00001F1D, 0x00000301},+{0x00001F20, 0x000003B7}, {0x00001F20, 0x00000313}, {0x00001F21, 0x000003B7}, {0x00001F21, 0x00000314},+{0x00001F22, 0x000003B7}, {0x00001F22, 0x00000313}, {0x00001F22, 0x00000300}, {0x00001F23, 0x000003B7},+{0x00001F23, 0x00000314}, {0x00001F23, 0x00000300}, {0x00001F24, 0x000003B7}, {0x00001F24, 0x00000313},+{0x00001F24, 0x00000301}, {0x00001F25, 0x000003B7}, {0x00001F25, 0x00000314}, {0x00001F25, 0x00000301},+{0x00001F26, 0x000003B7}, {0x00001F26, 0x00000313}, {0x00001F26, 0x00000342}, {0x00001F27, 0x000003B7},+{0x00001F27, 0x00000314}, {0x00001F27, 0x00000342}, {0x00001F28, 0x00000397}, {0x00001F28, 0x00000313},+{0x00001F29, 0x00000397}, {0x00001F29, 0x00000314}, {0x00001F2A, 0x00000397}, {0x00001F2A, 0x00000313},+{0x00001F2A, 0x00000300}, {0x00001F2B, 0x00000397}, {0x00001F2B, 0x00000314}, {0x00001F2B, 0x00000300},+{0x00001F2C, 0x00000397}, {0x00001F2C, 0x00000313}, {0x00001F2C, 0x00000301}, {0x00001F2D, 0x00000397},+{0x00001F2D, 0x00000314}, {0x00001F2D, 0x00000301}, {0x00001F2E, 0x00000397}, {0x00001F2E, 0x00000313},+{0x00001F2E, 0x00000342}, {0x00001F2F, 0x00000397}, {0x00001F2F, 0x00000314}, {0x00001F2F, 0x00000342},+{0x00001F30, 0x000003B9}, {0x00001F30, 0x00000313}, {0x00001F31, 0x000003B9}, {0x00001F31, 0x00000314},+{0x00001F32, 0x000003B9}, {0x00001F32, 0x00000313}, {0x00001F32, 0x00000300}, {0x00001F33, 0x000003B9},+{0x00001F33, 0x00000314}, {0x00001F33, 0x00000300}, {0x00001F34, 0x000003B9}, {0x00001F34, 0x00000313},+{0x00001F34, 0x00000301}, {0x00001F35, 0x000003B9}, {0x00001F35, 0x00000314}, {0x00001F35, 0x00000301},+{0x00001F36, 0x000003B9}, {0x00001F36, 0x00000313}, {0x00001F36, 0x00000342}, {0x00001F37, 0x000003B9},+{0x00001F37, 0x00000314}, {0x00001F37, 0x00000342}, {0x00001F38, 0x00000399}, {0x00001F38, 0x00000313},+{0x00001F39, 0x00000399}, {0x00001F39, 0x00000314}, {0x00001F3A, 0x00000399}, {0x00001F3A, 0x00000313},+{0x00001F3A, 0x00000300}, {0x00001F3B, 0x00000399}, {0x00001F3B, 0x00000314}, {0x00001F3B, 0x00000300},+{0x00001F3C, 0x00000399}, {0x00001F3C, 0x00000313}, {0x00001F3C, 0x00000301}, {0x00001F3D, 0x00000399},+{0x00001F3D, 0x00000314}, {0x00001F3D, 0x00000301}, {0x00001F3E, 0x00000399}, {0x00001F3E, 0x00000313},+{0x00001F3E, 0x00000342}, {0x00001F3F, 0x00000399}, {0x00001F3F, 0x00000314}, {0x00001F3F, 0x00000342},+{0x00001F40, 0x000003BF}, {0x00001F40, 0x00000313}, {0x00001F41, 0x000003BF}, {0x00001F41, 0x00000314},+{0x00001F42, 0x000003BF}, {0x00001F42, 0x00000313}, {0x00001F42, 0x00000300}, {0x00001F43, 0x000003BF},+{0x00001F43, 0x00000314}, {0x00001F43, 0x00000300}, {0x00001F44, 0x000003BF}, {0x00001F44, 0x00000313},+{0x00001F44, 0x00000301}, {0x00001F45, 0x000003BF}, {0x00001F45, 0x00000314}, {0x00001F45, 0x00000301},+{0x00001F48, 0x0000039F}, {0x00001F48, 0x00000313}, {0x00001F49, 0x0000039F}, {0x00001F49, 0x00000314},+{0x00001F4A, 0x0000039F}, {0x00001F4A, 0x00000313}, {0x00001F4A, 0x00000300}, {0x00001F4B, 0x0000039F},+{0x00001F4B, 0x00000314}, {0x00001F4B, 0x00000300}, {0x00001F4C, 0x0000039F}, {0x00001F4C, 0x00000313},+{0x00001F4C, 0x00000301}, {0x00001F4D, 0x0000039F}, {0x00001F4D, 0x00000314}, {0x00001F4D, 0x00000301},+{0x00001F50, 0x000003C5}, {0x00001F50, 0x00000313}, {0x00001F51, 0x000003C5}, {0x00001F51, 0x00000314},+{0x00001F52, 0x000003C5}, {0x00001F52, 0x00000313}, {0x00001F52, 0x00000300}, {0x00001F53, 0x000003C5},+{0x00001F53, 0x00000314}, {0x00001F53, 0x00000300}, {0x00001F54, 0x000003C5}, {0x00001F54, 0x00000313},+{0x00001F54, 0x00000301}, {0x00001F55, 0x000003C5}, {0x00001F55, 0x00000314}, {0x00001F55, 0x00000301},+{0x00001F56, 0x000003C5}, {0x00001F56, 0x00000313}, {0x00001F56, 0x00000342}, {0x00001F57, 0x000003C5},+{0x00001F57, 0x00000314}, {0x00001F57, 0x00000342}, {0x00001F59, 0x000003A5}, {0x00001F59, 0x00000314},+{0x00001F5B, 0x000003A5}, {0x00001F5B, 0x00000314}, {0x00001F5B, 0x00000300}, {0x00001F5D, 0x000003A5},+{0x00001F5D, 0x00000314}, {0x00001F5D, 0x00000301}, {0x00001F5F, 0x000003A5}, {0x00001F5F, 0x00000314},+{0x00001F5F, 0x00000342}, {0x00001F60, 0x000003C9}, {0x00001F60, 0x00000313}, {0x00001F61, 0x000003C9},+{0x00001F61, 0x00000314}, {0x00001F62, 0x000003C9}, {0x00001F62, 0x00000313}, {0x00001F62, 0x00000300},+{0x00001F63, 0x000003C9}, {0x00001F63, 0x00000314}, {0x00001F63, 0x00000300}, {0x00001F64, 0x000003C9},+{0x00001F64, 0x00000313}, {0x00001F64, 0x00000301}, {0x00001F65, 0x000003C9}, {0x00001F65, 0x00000314},+{0x00001F65, 0x00000301}, {0x00001F66, 0x000003C9}, {0x00001F66, 0x00000313}, {0x00001F66, 0x00000342},+{0x00001F67, 0x000003C9}, {0x00001F67, 0x00000314}, {0x00001F67, 0x00000342}, {0x00001F68, 0x000003A9},+{0x00001F68, 0x00000313}, {0x00001F69, 0x000003A9}, {0x00001F69, 0x00000314}, {0x00001F6A, 0x000003A9},+{0x00001F6A, 0x00000313}, {0x00001F6A, 0x00000300}, {0x00001F6B, 0x000003A9}, {0x00001F6B, 0x00000314},+{0x00001F6B, 0x00000300}, {0x00001F6C, 0x000003A9}, {0x00001F6C, 0x00000313}, {0x00001F6C, 0x00000301},+{0x00001F6D, 0x000003A9}, {0x00001F6D, 0x00000314}, {0x00001F6D, 0x00000301}, {0x00001F6E, 0x000003A9},+{0x00001F6E, 0x00000313}, {0x00001F6E, 0x00000342}, {0x00001F6F, 0x000003A9}, {0x00001F6F, 0x00000314},+{0x00001F6F, 0x00000342}, {0x00001F70, 0x000003B1}, {0x00001F70, 0x00000300}, {0x00001F71, 0x000003B1},+{0x00001F71, 0x00000301}, {0x00001F72, 0x000003B5}, {0x00001F72, 0x00000300}, {0x00001F73, 0x000003B5},+{0x00001F73, 0x00000301}, {0x00001F74, 0x000003B7}, {0x00001F74, 0x00000300}, {0x00001F75, 0x000003B7},+{0x00001F75, 0x00000301}, {0x00001F76, 0x000003B9}, {0x00001F76, 0x00000300}, {0x00001F77, 0x000003B9},+{0x00001F77, 0x00000301}, {0x00001F78, 0x000003BF}, {0x00001F78, 0x00000300}, {0x00001F79, 0x000003BF},+{0x00001F79, 0x00000301}, {0x00001F7A, 0x000003C5}, {0x00001F7A, 0x00000300}, {0x00001F7B, 0x000003C5},+{0x00001F7B, 0x00000301}, {0x00001F7C, 0x000003C9}, {0x00001F7C, 0x00000300}, {0x00001F7D, 0x000003C9},+{0x00001F7D, 0x00000301}, {0x00001F80, 0x000003B1}, {0x00001F80, 0x00000313}, {0x00001F80, 0x00000345},+{0x00001F81, 0x000003B1}, {0x00001F81, 0x00000314}, {0x00001F81, 0x00000345}, {0x00001F82, 0x000003B1},+{0x00001F82, 0x00000313}, {0x00001F82, 0x00000300}, {0x00001F82, 0x00000345}, {0x00001F83, 0x000003B1},+{0x00001F83, 0x00000314}, {0x00001F83, 0x00000300}, {0x00001F83, 0x00000345}, {0x00001F84, 0x000003B1},+{0x00001F84, 0x00000313}, {0x00001F84, 0x00000301}, {0x00001F84, 0x00000345}, {0x00001F85, 0x000003B1},+{0x00001F85, 0x00000314}, {0x00001F85, 0x00000301}, {0x00001F85, 0x00000345}, {0x00001F86, 0x000003B1},+{0x00001F86, 0x00000313}, {0x00001F86, 0x00000342}, {0x00001F86, 0x00000345}, {0x00001F87, 0x000003B1},+{0x00001F87, 0x00000314}, {0x00001F87, 0x00000342}, {0x00001F87, 0x00000345}, {0x00001F88, 0x00000391},+{0x00001F88, 0x00000313}, {0x00001F88, 0x00000345}, {0x00001F89, 0x00000391}, {0x00001F89, 0x00000314},+{0x00001F89, 0x00000345}, {0x00001F8A, 0x00000391}, {0x00001F8A, 0x00000313}, {0x00001F8A, 0x00000300},+{0x00001F8A, 0x00000345}, {0x00001F8B, 0x00000391}, {0x00001F8B, 0x00000314}, {0x00001F8B, 0x00000300},+{0x00001F8B, 0x00000345}, {0x00001F8C, 0x00000391}, {0x00001F8C, 0x00000313}, {0x00001F8C, 0x00000301},+{0x00001F8C, 0x00000345}, {0x00001F8D, 0x00000391}, {0x00001F8D, 0x00000314}, {0x00001F8D, 0x00000301},+{0x00001F8D, 0x00000345}, {0x00001F8E, 0x00000391}, {0x00001F8E, 0x00000313}, {0x00001F8E, 0x00000342},+{0x00001F8E, 0x00000345}, {0x00001F8F, 0x00000391}, {0x00001F8F, 0x00000314}, {0x00001F8F, 0x00000342},+{0x00001F8F, 0x00000345}, {0x00001F90, 0x000003B7}, {0x00001F90, 0x00000313}, {0x00001F90, 0x00000345},+{0x00001F91, 0x000003B7}, {0x00001F91, 0x00000314}, {0x00001F91, 0x00000345}, {0x00001F92, 0x000003B7},+{0x00001F92, 0x00000313}, {0x00001F92, 0x00000300}, {0x00001F92, 0x00000345}, {0x00001F93, 0x000003B7},+{0x00001F93, 0x00000314}, {0x00001F93, 0x00000300}, {0x00001F93, 0x00000345}, {0x00001F94, 0x000003B7},+{0x00001F94, 0x00000313}, {0x00001F94, 0x00000301}, {0x00001F94, 0x00000345}, {0x00001F95, 0x000003B7},+{0x00001F95, 0x00000314}, {0x00001F95, 0x00000301}, {0x00001F95, 0x00000345}, {0x00001F96, 0x000003B7},+{0x00001F96, 0x00000313}, {0x00001F96, 0x00000342}, {0x00001F96, 0x00000345}, {0x00001F97, 0x000003B7},+{0x00001F97, 0x00000314}, {0x00001F97, 0x00000342}, {0x00001F97, 0x00000345}, {0x00001F98, 0x00000397},+{0x00001F98, 0x00000313}, {0x00001F98, 0x00000345}, {0x00001F99, 0x00000397}, {0x00001F99, 0x00000314},+{0x00001F99, 0x00000345}, {0x00001F9A, 0x00000397}, {0x00001F9A, 0x00000313}, {0x00001F9A, 0x00000300},+{0x00001F9A, 0x00000345}, {0x00001F9B, 0x00000397}, {0x00001F9B, 0x00000314}, {0x00001F9B, 0x00000300},+{0x00001F9B, 0x00000345}, {0x00001F9C, 0x00000397}, {0x00001F9C, 0x00000313}, {0x00001F9C, 0x00000301},+{0x00001F9C, 0x00000345}, {0x00001F9D, 0x00000397}, {0x00001F9D, 0x00000314}, {0x00001F9D, 0x00000301},+{0x00001F9D, 0x00000345}, {0x00001F9E, 0x00000397}, {0x00001F9E, 0x00000313}, {0x00001F9E, 0x00000342},+{0x00001F9E, 0x00000345}, {0x00001F9F, 0x00000397}, {0x00001F9F, 0x00000314}, {0x00001F9F, 0x00000342},+{0x00001F9F, 0x00000345}, {0x00001FA0, 0x000003C9}, {0x00001FA0, 0x00000313}, {0x00001FA0, 0x00000345},+{0x00001FA1, 0x000003C9}, {0x00001FA1, 0x00000314}, {0x00001FA1, 0x00000345}, {0x00001FA2, 0x000003C9},+{0x00001FA2, 0x00000313}, {0x00001FA2, 0x00000300}, {0x00001FA2, 0x00000345}, {0x00001FA3, 0x000003C9},+{0x00001FA3, 0x00000314}, {0x00001FA3, 0x00000300}, {0x00001FA3, 0x00000345}, {0x00001FA4, 0x000003C9},+{0x00001FA4, 0x00000313}, {0x00001FA4, 0x00000301}, {0x00001FA4, 0x00000345}, {0x00001FA5, 0x000003C9},+{0x00001FA5, 0x00000314}, {0x00001FA5, 0x00000301}, {0x00001FA5, 0x00000345}, {0x00001FA6, 0x000003C9},+{0x00001FA6, 0x00000313}, {0x00001FA6, 0x00000342}, {0x00001FA6, 0x00000345}, {0x00001FA7, 0x000003C9},+{0x00001FA7, 0x00000314}, {0x00001FA7, 0x00000342}, {0x00001FA7, 0x00000345}, {0x00001FA8, 0x000003A9},+{0x00001FA8, 0x00000313}, {0x00001FA8, 0x00000345}, {0x00001FA9, 0x000003A9}, {0x00001FA9, 0x00000314},+{0x00001FA9, 0x00000345}, {0x00001FAA, 0x000003A9}, {0x00001FAA, 0x00000313}, {0x00001FAA, 0x00000300},+{0x00001FAA, 0x00000345}, {0x00001FAB, 0x000003A9}, {0x00001FAB, 0x00000314}, {0x00001FAB, 0x00000300},+{0x00001FAB, 0x00000345}, {0x00001FAC, 0x000003A9}, {0x00001FAC, 0x00000313}, {0x00001FAC, 0x00000301},+{0x00001FAC, 0x00000345}, {0x00001FAD, 0x000003A9}, {0x00001FAD, 0x00000314}, {0x00001FAD, 0x00000301},+{0x00001FAD, 0x00000345}, {0x00001FAE, 0x000003A9}, {0x00001FAE, 0x00000313}, {0x00001FAE, 0x00000342},+{0x00001FAE, 0x00000345}, {0x00001FAF, 0x000003A9}, {0x00001FAF, 0x00000314}, {0x00001FAF, 0x00000342},+{0x00001FAF, 0x00000345}, {0x00001FB0, 0x000003B1}, {0x00001FB0, 0x00000306}, {0x00001FB1, 0x000003B1},+{0x00001FB1, 0x00000304}, {0x00001FB2, 0x000003B1}, {0x00001FB2, 0x00000300}, {0x00001FB2, 0x00000345},+{0x00001FB3, 0x000003B1}, {0x00001FB3, 0x00000345}, {0x00001FB4, 0x000003B1}, {0x00001FB4, 0x00000301},+{0x00001FB4, 0x00000345}, {0x00001FB6, 0x000003B1}, {0x00001FB6, 0x00000342}, {0x00001FB7, 0x000003B1},+{0x00001FB7, 0x00000342}, {0x00001FB7, 0x00000345}, {0x00001FB8, 0x00000391}, {0x00001FB8, 0x00000306},+{0x00001FB9, 0x00000391}, {0x00001FB9, 0x00000304}, {0x00001FBA, 0x00000391}, {0x00001FBA, 0x00000300},+{0x00001FBB, 0x00000391}, {0x00001FBB, 0x00000301}, {0x00001FBC, 0x00000391}, {0x00001FBC, 0x00000345},+{0x00001FBE, 0x000003B9}, {0x00001FC1, 0x000000A8}, {0x00001FC1, 0x00000342}, {0x00001FC2, 0x000003B7},+{0x00001FC2, 0x00000300}, {0x00001FC2, 0x00000345}, {0x00001FC3, 0x000003B7}, {0x00001FC3, 0x00000345},+{0x00001FC4, 0x000003B7}, {0x00001FC4, 0x00000301}, {0x00001FC4, 0x00000345}, {0x00001FC6, 0x000003B7},+{0x00001FC6, 0x00000342}, {0x00001FC7, 0x000003B7}, {0x00001FC7, 0x00000342}, {0x00001FC7, 0x00000345},+{0x00001FC8, 0x00000395}, {0x00001FC8, 0x00000300}, {0x00001FC9, 0x00000395}, {0x00001FC9, 0x00000301},+{0x00001FCA, 0x00000397}, {0x00001FCA, 0x00000300}, {0x00001FCB, 0x00000397}, {0x00001FCB, 0x00000301},+{0x00001FCC, 0x00000397}, {0x00001FCC, 0x00000345}, {0x00001FCD, 0x00001FBF}, {0x00001FCD, 0x00000300},+{0x00001FCE, 0x00001FBF}, {0x00001FCE, 0x00000301}, {0x00001FCF, 0x00001FBF}, {0x00001FCF, 0x00000342},+{0x00001FD0, 0x000003B9}, {0x00001FD0, 0x00000306}, {0x00001FD1, 0x000003B9}, {0x00001FD1, 0x00000304},+{0x00001FD2, 0x000003B9}, {0x00001FD2, 0x00000308}, {0x00001FD2, 0x00000300}, {0x00001FD3, 0x000003B9},+{0x00001FD3, 0x00000308}, {0x00001FD3, 0x00000301}, {0x00001FD6, 0x000003B9}, {0x00001FD6, 0x00000342},+{0x00001FD7, 0x000003B9}, {0x00001FD7, 0x00000308}, {0x00001FD7, 0x00000342}, {0x00001FD8, 0x00000399},+{0x00001FD8, 0x00000306}, {0x00001FD9, 0x00000399}, {0x00001FD9, 0x00000304}, {0x00001FDA, 0x00000399},+{0x00001FDA, 0x00000300}, {0x00001FDB, 0x00000399}, {0x00001FDB, 0x00000301}, {0x00001FDD, 0x00001FFE},+{0x00001FDD, 0x00000300}, {0x00001FDE, 0x00001FFE}, {0x00001FDE, 0x00000301}, {0x00001FDF, 0x00001FFE},+{0x00001FDF, 0x00000342}, {0x00001FE0, 0x000003C5}, {0x00001FE0, 0x00000306}, {0x00001FE1, 0x000003C5},+{0x00001FE1, 0x00000304}, {0x00001FE2, 0x000003C5}, {0x00001FE2, 0x00000308}, {0x00001FE2, 0x00000300},+{0x00001FE3, 0x000003C5}, {0x00001FE3, 0x00000308}, {0x00001FE3, 0x00000301}, {0x00001FE4, 0x000003C1},+{0x00001FE4, 0x00000313}, {0x00001FE5, 0x000003C1}, {0x00001FE5, 0x00000314}, {0x00001FE6, 0x000003C5},+{0x00001FE6, 0x00000342}, {0x00001FE7, 0x000003C5}, {0x00001FE7, 0x00000308}, {0x00001FE7, 0x00000342},+{0x00001FE8, 0x000003A5}, {0x00001FE8, 0x00000306}, {0x00001FE9, 0x000003A5}, {0x00001FE9, 0x00000304},+{0x00001FEA, 0x000003A5}, {0x00001FEA, 0x00000300}, {0x00001FEB, 0x000003A5}, {0x00001FEB, 0x00000301},+{0x00001FEC, 0x000003A1}, {0x00001FEC, 0x00000314}, {0x00001FED, 0x000000A8}, {0x00001FED, 0x00000300},+{0x00001FEE, 0x000000A8}, {0x00001FEE, 0x00000301}, {0x00001FEF, 0x00000060}, {0x00001FF2, 0x000003C9},+{0x00001FF2, 0x00000300}, {0x00001FF2, 0x00000345}, {0x00001FF3, 0x000003C9}, {0x00001FF3, 0x00000345},+{0x00001FF4, 0x000003C9}, {0x00001FF4, 0x00000301}, {0x00001FF4, 0x00000345}, {0x00001FF6, 0x000003C9},+{0x00001FF6, 0x00000342}, {0x00001FF7, 0x000003C9}, {0x00001FF7, 0x00000342}, {0x00001FF7, 0x00000345},+{0x00001FF8, 0x0000039F}, {0x00001FF8, 0x00000300}, {0x00001FF9, 0x0000039F}, {0x00001FF9, 0x00000301},+{0x00001FFA, 0x000003A9}, {0x00001FFA, 0x00000300}, {0x00001FFB, 0x000003A9}, {0x00001FFB, 0x00000301},+{0x00001FFC, 0x000003A9}, {0x00001FFC, 0x00000345}, {0x00001FFD, 0x000000B4}, {0x00002000, 0x00002002},+{0x00002001, 0x00002003}, {0x00002126, 0x000003A9}, {0x0000212A, 0x0000004B}, {0x0000212B, 0x00000041},+{0x0000212B, 0x0000030A}, {0x0000219A, 0x00002190}, {0x0000219A, 0x00000338}, {0x0000219B, 0x00002192},+{0x0000219B, 0x00000338}, {0x000021AE, 0x00002194}, {0x000021AE, 0x00000338}, {0x000021CD, 0x000021D0},+{0x000021CD, 0x00000338}, {0x000021CE, 0x000021D4}, {0x000021CE, 0x00000338}, {0x000021CF, 0x000021D2},+{0x000021CF, 0x00000338}, {0x00002204, 0x00002203}, {0x00002204, 0x00000338}, {0x00002209, 0x00002208},+{0x00002209, 0x00000338}, {0x0000220C, 0x0000220B}, {0x0000220C, 0x00000338}, {0x00002224, 0x00002223},+{0x00002224, 0x00000338}, {0x00002226, 0x00002225}, {0x00002226, 0x00000338}, {0x00002241, 0x0000223C},+{0x00002241, 0x00000338}, {0x00002244, 0x00002243}, {0x00002244, 0x00000338}, {0x00002247, 0x00002245},+{0x00002247, 0x00000338}, {0x00002249, 0x00002248}, {0x00002249, 0x00000338}, {0x00002260, 0x0000003D},+{0x00002260, 0x00000338}, {0x00002262, 0x00002261}, {0x00002262, 0x00000338}, {0x0000226D, 0x0000224D},+{0x0000226D, 0x00000338}, {0x0000226E, 0x0000003C}, {0x0000226E, 0x00000338}, {0x0000226F, 0x0000003E},+{0x0000226F, 0x00000338}, {0x00002270, 0x00002264}, {0x00002270, 0x00000338}, {0x00002271, 0x00002265},+{0x00002271, 0x00000338}, {0x00002274, 0x00002272}, {0x00002274, 0x00000338}, {0x00002275, 0x00002273},+{0x00002275, 0x00000338}, {0x00002278, 0x00002276}, {0x00002278, 0x00000338}, {0x00002279, 0x00002277},+{0x00002279, 0x00000338}, {0x00002280, 0x0000227A}, {0x00002280, 0x00000338}, {0x00002281, 0x0000227B},+{0x00002281, 0x00000338}, {0x00002284, 0x00002282}, {0x00002284, 0x00000338}, {0x00002285, 0x00002283},+{0x00002285, 0x00000338}, {0x00002288, 0x00002286}, {0x00002288, 0x00000338}, {0x00002289, 0x00002287},+{0x00002289, 0x00000338}, {0x000022AC, 0x000022A2}, {0x000022AC, 0x00000338}, {0x000022AD, 0x000022A8},+{0x000022AD, 0x00000338}, {0x000022AE, 0x000022A9}, {0x000022AE, 0x00000338}, {0x000022AF, 0x000022AB},+{0x000022AF, 0x00000338}, {0x000022E0, 0x0000227C}, {0x000022E0, 0x00000338}, {0x000022E1, 0x0000227D},+{0x000022E1, 0x00000338}, {0x000022E2, 0x00002291}, {0x000022E2, 0x00000338}, {0x000022E3, 0x00002292},+{0x000022E3, 0x00000338}, {0x000022EA, 0x000022B2}, {0x000022EA, 0x00000338}, {0x000022EB, 0x000022B3},+{0x000022EB, 0x00000338}, {0x000022EC, 0x000022B4}, {0x000022EC, 0x00000338}, {0x000022ED, 0x000022B5},+{0x000022ED, 0x00000338}, {0x00002329, 0x00003008}, {0x0000232A, 0x00003009}, {0x00002ADC, 0x00002ADD},+{0x00002ADC, 0x00000338}, {0x0000304C, 0x0000304B}, {0x0000304C, 0x00003099}, {0x0000304E, 0x0000304D},+{0x0000304E, 0x00003099}, {0x00003050, 0x0000304F}, {0x00003050, 0x00003099}, {0x00003052, 0x00003051},+{0x00003052, 0x00003099}, {0x00003054, 0x00003053}, {0x00003054, 0x00003099}, {0x00003056, 0x00003055},+{0x00003056, 0x00003099}, {0x00003058, 0x00003057}, {0x00003058, 0x00003099}, {0x0000305A, 0x00003059},+{0x0000305A, 0x00003099}, {0x0000305C, 0x0000305B}, {0x0000305C, 0x00003099}, {0x0000305E, 0x0000305D},+{0x0000305E, 0x00003099}, {0x00003060, 0x0000305F}, {0x00003060, 0x00003099}, {0x00003062, 0x00003061},+{0x00003062, 0x00003099}, {0x00003065, 0x00003064}, {0x00003065, 0x00003099}, {0x00003067, 0x00003066},+{0x00003067, 0x00003099}, {0x00003069, 0x00003068}, {0x00003069, 0x00003099}, {0x00003070, 0x0000306F},+{0x00003070, 0x00003099}, {0x00003071, 0x0000306F}, {0x00003071, 0x0000309A}, {0x00003073, 0x00003072},+{0x00003073, 0x00003099}, {0x00003074, 0x00003072}, {0x00003074, 0x0000309A}, {0x00003076, 0x00003075},+{0x00003076, 0x00003099}, {0x00003077, 0x00003075}, {0x00003077, 0x0000309A}, {0x00003079, 0x00003078},+{0x00003079, 0x00003099}, {0x0000307A, 0x00003078}, {0x0000307A, 0x0000309A}, {0x0000307C, 0x0000307B},+{0x0000307C, 0x00003099}, {0x0000307D, 0x0000307B}, {0x0000307D, 0x0000309A}, {0x00003094, 0x00003046},+{0x00003094, 0x00003099}, {0x0000309E, 0x0000309D}, {0x0000309E, 0x00003099}, {0x000030AC, 0x000030AB},+{0x000030AC, 0x00003099}, {0x000030AE, 0x000030AD}, {0x000030AE, 0x00003099}, {0x000030B0, 0x000030AF},+{0x000030B0, 0x00003099}, {0x000030B2, 0x000030B1}, {0x000030B2, 0x00003099}, {0x000030B4, 0x000030B3},+{0x000030B4, 0x00003099}, {0x000030B6, 0x000030B5}, {0x000030B6, 0x00003099}, {0x000030B8, 0x000030B7},+{0x000030B8, 0x00003099}, {0x000030BA, 0x000030B9}, {0x000030BA, 0x00003099}, {0x000030BC, 0x000030BB},+{0x000030BC, 0x00003099}, {0x000030BE, 0x000030BD}, {0x000030BE, 0x00003099}, {0x000030C0, 0x000030BF},+{0x000030C0, 0x00003099}, {0x000030C2, 0x000030C1}, {0x000030C2, 0x00003099}, {0x000030C5, 0x000030C4},+{0x000030C5, 0x00003099}, {0x000030C7, 0x000030C6}, {0x000030C7, 0x00003099}, {0x000030C9, 0x000030C8},+{0x000030C9, 0x00003099}, {0x000030D0, 0x000030CF}, {0x000030D0, 0x00003099}, {0x000030D1, 0x000030CF},+{0x000030D1, 0x0000309A}, {0x000030D3, 0x000030D2}, {0x000030D3, 0x00003099}, {0x000030D4, 0x000030D2},+{0x000030D4, 0x0000309A}, {0x000030D6, 0x000030D5}, {0x000030D6, 0x00003099}, {0x000030D7, 0x000030D5},+{0x000030D7, 0x0000309A}, {0x000030D9, 0x000030D8}, {0x000030D9, 0x00003099}, {0x000030DA, 0x000030D8},+{0x000030DA, 0x0000309A}, {0x000030DC, 0x000030DB}, {0x000030DC, 0x00003099}, {0x000030DD, 0x000030DB},+{0x000030DD, 0x0000309A}, {0x000030F4, 0x000030A6}, {0x000030F4, 0x00003099}, {0x000030F7, 0x000030EF},+{0x000030F7, 0x00003099}, {0x000030F8, 0x000030F0}, {0x000030F8, 0x00003099}, {0x000030F9, 0x000030F1},+{0x000030F9, 0x00003099}, {0x000030FA, 0x000030F2}, {0x000030FA, 0x00003099}, {0x000030FE, 0x000030FD},+{0x000030FE, 0x00003099}, {0x0000F900, 0x00008C48}, {0x0000F901, 0x000066F4}, {0x0000F902, 0x00008ECA},+{0x0000F903, 0x00008CC8}, {0x0000F904, 0x00006ED1}, {0x0000F905, 0x00004E32}, {0x0000F906, 0x000053E5},+{0x0000F907, 0x00009F9C}, {0x0000F908, 0x00009F9C}, {0x0000F909, 0x00005951}, {0x0000F90A, 0x000091D1},+{0x0000F90B, 0x00005587}, {0x0000F90C, 0x00005948}, {0x0000F90D, 0x000061F6}, {0x0000F90E, 0x00007669},+{0x0000F90F, 0x00007F85}, {0x0000F910, 0x0000863F}, {0x0000F911, 0x000087BA}, {0x0000F912, 0x000088F8},+{0x0000F913, 0x0000908F}, {0x0000F914, 0x00006A02}, {0x0000F915, 0x00006D1B}, {0x0000F916, 0x000070D9},+{0x0000F917, 0x000073DE}, {0x0000F918, 0x0000843D}, {0x0000F919, 0x0000916A}, {0x0000F91A, 0x000099F1},+{0x0000F91B, 0x00004E82}, {0x0000F91C, 0x00005375}, {0x0000F91D, 0x00006B04}, {0x0000F91E, 0x0000721B},+{0x0000F91F, 0x0000862D}, {0x0000F920, 0x00009E1E}, {0x0000F921, 0x00005D50}, {0x0000F922, 0x00006FEB},+{0x0000F923, 0x000085CD}, {0x0000F924, 0x00008964}, {0x0000F925, 0x000062C9}, {0x0000F926, 0x000081D8},+{0x0000F927, 0x0000881F}, {0x0000F928, 0x00005ECA}, {0x0000F929, 0x00006717}, {0x0000F92A, 0x00006D6A},+{0x0000F92B, 0x000072FC}, {0x0000F92C, 0x000090CE}, {0x0000F92D, 0x00004F86}, {0x0000F92E, 0x000051B7},+{0x0000F92F, 0x000052DE}, {0x0000F930, 0x000064C4}, {0x0000F931, 0x00006AD3}, {0x0000F932, 0x00007210},+{0x0000F933, 0x000076E7}, {0x0000F934, 0x00008001}, {0x0000F935, 0x00008606}, {0x0000F936, 0x0000865C},+{0x0000F937, 0x00008DEF}, {0x0000F938, 0x00009732}, {0x0000F939, 0x00009B6F}, {0x0000F93A, 0x00009DFA},+{0x0000F93B, 0x0000788C}, {0x0000F93C, 0x0000797F}, {0x0000F93D, 0x00007DA0}, {0x0000F93E, 0x000083C9},+{0x0000F93F, 0x00009304}, {0x0000F940, 0x00009E7F}, {0x0000F941, 0x00008AD6}, {0x0000F942, 0x000058DF},+{0x0000F943, 0x00005F04}, {0x0000F944, 0x00007C60}, {0x0000F945, 0x0000807E}, {0x0000F946, 0x00007262},+{0x0000F947, 0x000078CA}, {0x0000F948, 0x00008CC2}, {0x0000F949, 0x000096F7}, {0x0000F94A, 0x000058D8},+{0x0000F94B, 0x00005C62}, {0x0000F94C, 0x00006A13}, {0x0000F94D, 0x00006DDA}, {0x0000F94E, 0x00006F0F},+{0x0000F94F, 0x00007D2F}, {0x0000F950, 0x00007E37}, {0x0000F951, 0x0000964B}, {0x0000F952, 0x000052D2},+{0x0000F953, 0x0000808B}, {0x0000F954, 0x000051DC}, {0x0000F955, 0x000051CC}, {0x0000F956, 0x00007A1C},+{0x0000F957, 0x00007DBE}, {0x0000F958, 0x000083F1}, {0x0000F959, 0x00009675}, {0x0000F95A, 0x00008B80},+{0x0000F95B, 0x000062CF}, {0x0000F95C, 0x00006A02}, {0x0000F95D, 0x00008AFE}, {0x0000F95E, 0x00004E39},+{0x0000F95F, 0x00005BE7}, {0x0000F960, 0x00006012}, {0x0000F961, 0x00007387}, {0x0000F962, 0x00007570},+{0x0000F963, 0x00005317}, {0x0000F964, 0x000078FB}, {0x0000F965, 0x00004FBF}, {0x0000F966, 0x00005FA9},+{0x0000F967, 0x00004E0D}, {0x0000F968, 0x00006CCC}, {0x0000F969, 0x00006578}, {0x0000F96A, 0x00007D22},+{0x0000F96B, 0x000053C3}, {0x0000F96C, 0x0000585E}, {0x0000F96D, 0x00007701}, {0x0000F96E, 0x00008449},+{0x0000F96F, 0x00008AAA}, {0x0000F970, 0x00006BBA}, {0x0000F971, 0x00008FB0}, {0x0000F972, 0x00006C88},+{0x0000F973, 0x000062FE}, {0x0000F974, 0x000082E5}, {0x0000F975, 0x000063A0}, {0x0000F976, 0x00007565},+{0x0000F977, 0x00004EAE}, {0x0000F978, 0x00005169}, {0x0000F979, 0x000051C9}, {0x0000F97A, 0x00006881},+{0x0000F97B, 0x00007CE7}, {0x0000F97C, 0x0000826F}, {0x0000F97D, 0x00008AD2}, {0x0000F97E, 0x000091CF},+{0x0000F97F, 0x000052F5}, {0x0000F980, 0x00005442}, {0x0000F981, 0x00005973}, {0x0000F982, 0x00005EEC},+{0x0000F983, 0x000065C5}, {0x0000F984, 0x00006FFE}, {0x0000F985, 0x0000792A}, {0x0000F986, 0x000095AD},+{0x0000F987, 0x00009A6A}, {0x0000F988, 0x00009E97}, {0x0000F989, 0x00009ECE}, {0x0000F98A, 0x0000529B},+{0x0000F98B, 0x000066C6}, {0x0000F98C, 0x00006B77}, {0x0000F98D, 0x00008F62}, {0x0000F98E, 0x00005E74},+{0x0000F98F, 0x00006190}, {0x0000F990, 0x00006200}, {0x0000F991, 0x0000649A}, {0x0000F992, 0x00006F23},+{0x0000F993, 0x00007149}, {0x0000F994, 0x00007489}, {0x0000F995, 0x000079CA}, {0x0000F996, 0x00007DF4},+{0x0000F997, 0x0000806F}, {0x0000F998, 0x00008F26}, {0x0000F999, 0x000084EE}, {0x0000F99A, 0x00009023},+{0x0000F99B, 0x0000934A}, {0x0000F99C, 0x00005217}, {0x0000F99D, 0x000052A3}, {0x0000F99E, 0x000054BD},+{0x0000F99F, 0x000070C8}, {0x0000F9A0, 0x000088C2}, {0x0000F9A1, 0x00008AAA}, {0x0000F9A2, 0x00005EC9},+{0x0000F9A3, 0x00005FF5}, {0x0000F9A4, 0x0000637B}, {0x0000F9A5, 0x00006BAE}, {0x0000F9A6, 0x00007C3E},+{0x0000F9A7, 0x00007375}, {0x0000F9A8, 0x00004EE4}, {0x0000F9A9, 0x000056F9}, {0x0000F9AA, 0x00005BE7},+{0x0000F9AB, 0x00005DBA}, {0x0000F9AC, 0x0000601C}, {0x0000F9AD, 0x000073B2}, {0x0000F9AE, 0x00007469},+{0x0000F9AF, 0x00007F9A}, {0x0000F9B0, 0x00008046}, {0x0000F9B1, 0x00009234}, {0x0000F9B2, 0x000096F6},+{0x0000F9B3, 0x00009748}, {0x0000F9B4, 0x00009818}, {0x0000F9B5, 0x00004F8B}, {0x0000F9B6, 0x000079AE},+{0x0000F9B7, 0x000091B4}, {0x0000F9B8, 0x000096B8}, {0x0000F9B9, 0x000060E1}, {0x0000F9BA, 0x00004E86},+{0x0000F9BB, 0x000050DA}, {0x0000F9BC, 0x00005BEE}, {0x0000F9BD, 0x00005C3F}, {0x0000F9BE, 0x00006599},+{0x0000F9BF, 0x00006A02}, {0x0000F9C0, 0x000071CE}, {0x0000F9C1, 0x00007642}, {0x0000F9C2, 0x000084FC},+{0x0000F9C3, 0x0000907C}, {0x0000F9C4, 0x00009F8D}, {0x0000F9C5, 0x00006688}, {0x0000F9C6, 0x0000962E},+{0x0000F9C7, 0x00005289}, {0x0000F9C8, 0x0000677B}, {0x0000F9C9, 0x000067F3}, {0x0000F9CA, 0x00006D41},+{0x0000F9CB, 0x00006E9C}, {0x0000F9CC, 0x00007409}, {0x0000F9CD, 0x00007559}, {0x0000F9CE, 0x0000786B},+{0x0000F9CF, 0x00007D10}, {0x0000F9D0, 0x0000985E}, {0x0000F9D1, 0x0000516D}, {0x0000F9D2, 0x0000622E},+{0x0000F9D3, 0x00009678}, {0x0000F9D4, 0x0000502B}, {0x0000F9D5, 0x00005D19}, {0x0000F9D6, 0x00006DEA},+{0x0000F9D7, 0x00008F2A}, {0x0000F9D8, 0x00005F8B}, {0x0000F9D9, 0x00006144}, {0x0000F9DA, 0x00006817},+{0x0000F9DB, 0x00007387}, {0x0000F9DC, 0x00009686}, {0x0000F9DD, 0x00005229}, {0x0000F9DE, 0x0000540F},+{0x0000F9DF, 0x00005C65}, {0x0000F9E0, 0x00006613}, {0x0000F9E1, 0x0000674E}, {0x0000F9E2, 0x000068A8},+{0x0000F9E3, 0x00006CE5}, {0x0000F9E4, 0x00007406}, {0x0000F9E5, 0x000075E2}, {0x0000F9E6, 0x00007F79},+{0x0000F9E7, 0x000088CF}, {0x0000F9E8, 0x000088E1}, {0x0000F9E9, 0x000091CC}, {0x0000F9EA, 0x000096E2},+{0x0000F9EB, 0x0000533F}, {0x0000F9EC, 0x00006EBA}, {0x0000F9ED, 0x0000541D}, {0x0000F9EE, 0x000071D0},+{0x0000F9EF, 0x00007498}, {0x0000F9F0, 0x000085FA}, {0x0000F9F1, 0x000096A3}, {0x0000F9F2, 0x00009C57},+{0x0000F9F3, 0x00009E9F}, {0x0000F9F4, 0x00006797}, {0x0000F9F5, 0x00006DCB}, {0x0000F9F6, 0x000081E8},+{0x0000F9F7, 0x00007ACB}, {0x0000F9F8, 0x00007B20}, {0x0000F9F9, 0x00007C92}, {0x0000F9FA, 0x000072C0},+{0x0000F9FB, 0x00007099}, {0x0000F9FC, 0x00008B58}, {0x0000F9FD, 0x00004EC0}, {0x0000F9FE, 0x00008336},+{0x0000F9FF, 0x0000523A}, {0x0000FA00, 0x00005207}, {0x0000FA01, 0x00005EA6}, {0x0000FA02, 0x000062D3},+{0x0000FA03, 0x00007CD6}, {0x0000FA04, 0x00005B85}, {0x0000FA05, 0x00006D1E}, {0x0000FA06, 0x000066B4},+{0x0000FA07, 0x00008F3B}, {0x0000FA08, 0x0000884C}, {0x0000FA09, 0x0000964D}, {0x0000FA0A, 0x0000898B},+{0x0000FA0B, 0x00005ED3}, {0x0000FA0C, 0x00005140}, {0x0000FA0D, 0x000055C0}, {0x0000FA10, 0x0000585A},+{0x0000FA12, 0x00006674}, {0x0000FA15, 0x000051DE}, {0x0000FA16, 0x0000732A}, {0x0000FA17, 0x000076CA},+{0x0000FA18, 0x0000793C}, {0x0000FA19, 0x0000795E}, {0x0000FA1A, 0x00007965}, {0x0000FA1B, 0x0000798F},+{0x0000FA1C, 0x00009756}, {0x0000FA1D, 0x00007CBE}, {0x0000FA1E, 0x00007FBD}, {0x0000FA20, 0x00008612},+{0x0000FA22, 0x00008AF8}, {0x0000FA25, 0x00009038}, {0x0000FA26, 0x000090FD}, {0x0000FA2A, 0x000098EF},+{0x0000FA2B, 0x000098FC}, {0x0000FA2C, 0x00009928}, {0x0000FA2D, 0x00009DB4}, {0x0000FA2E, 0x000090DE},+{0x0000FA2F, 0x000096B7}, {0x0000FA30, 0x00004FAE}, {0x0000FA31, 0x000050E7}, {0x0000FA32, 0x0000514D},+{0x0000FA33, 0x000052C9}, {0x0000FA34, 0x000052E4}, {0x0000FA35, 0x00005351}, {0x0000FA36, 0x0000559D},+{0x0000FA37, 0x00005606}, {0x0000FA38, 0x00005668}, {0x0000FA39, 0x00005840}, {0x0000FA3A, 0x000058A8},+{0x0000FA3B, 0x00005C64}, {0x0000FA3C, 0x00005C6E}, {0x0000FA3D, 0x00006094}, {0x0000FA3E, 0x00006168},+{0x0000FA3F, 0x0000618E}, {0x0000FA40, 0x000061F2}, {0x0000FA41, 0x0000654F}, {0x0000FA42, 0x000065E2},+{0x0000FA43, 0x00006691}, {0x0000FA44, 0x00006885}, {0x0000FA45, 0x00006D77}, {0x0000FA46, 0x00006E1A},+{0x0000FA47, 0x00006F22}, {0x0000FA48, 0x0000716E}, {0x0000FA49, 0x0000722B}, {0x0000FA4A, 0x00007422},+{0x0000FA4B, 0x00007891}, {0x0000FA4C, 0x0000793E}, {0x0000FA4D, 0x00007949}, {0x0000FA4E, 0x00007948},+{0x0000FA4F, 0x00007950}, {0x0000FA50, 0x00007956}, {0x0000FA51, 0x0000795D}, {0x0000FA52, 0x0000798D},+{0x0000FA53, 0x0000798E}, {0x0000FA54, 0x00007A40}, {0x0000FA55, 0x00007A81}, {0x0000FA56, 0x00007BC0},+{0x0000FA57, 0x00007DF4}, {0x0000FA58, 0x00007E09}, {0x0000FA59, 0x00007E41}, {0x0000FA5A, 0x00007F72},+{0x0000FA5B, 0x00008005}, {0x0000FA5C, 0x000081ED}, {0x0000FA5D, 0x00008279}, {0x0000FA5E, 0x00008279},+{0x0000FA5F, 0x00008457}, {0x0000FA60, 0x00008910}, {0x0000FA61, 0x00008996}, {0x0000FA62, 0x00008B01},+{0x0000FA63, 0x00008B39}, {0x0000FA64, 0x00008CD3}, {0x0000FA65, 0x00008D08}, {0x0000FA66, 0x00008FB6},+{0x0000FA67, 0x00009038}, {0x0000FA68, 0x000096E3}, {0x0000FA69, 0x000097FF}, {0x0000FA6A, 0x0000983B},+{0x0000FA6B, 0x00006075}, {0x0000FA6C, 0x000242EE}, {0x0000FA6D, 0x00008218}, {0x0000FA70, 0x00004E26},+{0x0000FA71, 0x000051B5}, {0x0000FA72, 0x00005168}, {0x0000FA73, 0x00004F80}, {0x0000FA74, 0x00005145},+{0x0000FA75, 0x00005180}, {0x0000FA76, 0x000052C7}, {0x0000FA77, 0x000052FA}, {0x0000FA78, 0x0000559D},+{0x0000FA79, 0x00005555}, {0x0000FA7A, 0x00005599}, {0x0000FA7B, 0x000055E2}, {0x0000FA7C, 0x0000585A},+{0x0000FA7D, 0x000058B3}, {0x0000FA7E, 0x00005944}, {0x0000FA7F, 0x00005954}, {0x0000FA80, 0x00005A62},+{0x0000FA81, 0x00005B28}, {0x0000FA82, 0x00005ED2}, {0x0000FA83, 0x00005ED9}, {0x0000FA84, 0x00005F69},+{0x0000FA85, 0x00005FAD}, {0x0000FA86, 0x000060D8}, {0x0000FA87, 0x0000614E}, {0x0000FA88, 0x00006108},+{0x0000FA89, 0x0000618E}, {0x0000FA8A, 0x00006160}, {0x0000FA8B, 0x000061F2}, {0x0000FA8C, 0x00006234},+{0x0000FA8D, 0x000063C4}, {0x0000FA8E, 0x0000641C}, {0x0000FA8F, 0x00006452}, {0x0000FA90, 0x00006556},+{0x0000FA91, 0x00006674}, {0x0000FA92, 0x00006717}, {0x0000FA93, 0x0000671B}, {0x0000FA94, 0x00006756},+{0x0000FA95, 0x00006B79}, {0x0000FA96, 0x00006BBA}, {0x0000FA97, 0x00006D41}, {0x0000FA98, 0x00006EDB},+{0x0000FA99, 0x00006ECB}, {0x0000FA9A, 0x00006F22}, {0x0000FA9B, 0x0000701E}, {0x0000FA9C, 0x0000716E},+{0x0000FA9D, 0x000077A7}, {0x0000FA9E, 0x00007235}, {0x0000FA9F, 0x000072AF}, {0x0000FAA0, 0x0000732A},+{0x0000FAA1, 0x00007471}, {0x0000FAA2, 0x00007506}, {0x0000FAA3, 0x0000753B}, {0x0000FAA4, 0x0000761D},+{0x0000FAA5, 0x0000761F}, {0x0000FAA6, 0x000076CA}, {0x0000FAA7, 0x000076DB}, {0x0000FAA8, 0x000076F4},+{0x0000FAA9, 0x0000774A}, {0x0000FAAA, 0x00007740}, {0x0000FAAB, 0x000078CC}, {0x0000FAAC, 0x00007AB1},+{0x0000FAAD, 0x00007BC0}, {0x0000FAAE, 0x00007C7B}, {0x0000FAAF, 0x00007D5B}, {0x0000FAB0, 0x00007DF4},+{0x0000FAB1, 0x00007F3E}, {0x0000FAB2, 0x00008005}, {0x0000FAB3, 0x00008352}, {0x0000FAB4, 0x000083EF},+{0x0000FAB5, 0x00008779}, {0x0000FAB6, 0x00008941}, {0x0000FAB7, 0x00008986}, {0x0000FAB8, 0x00008996},+{0x0000FAB9, 0x00008ABF}, {0x0000FABA, 0x00008AF8}, {0x0000FABB, 0x00008ACB}, {0x0000FABC, 0x00008B01},+{0x0000FABD, 0x00008AFE}, {0x0000FABE, 0x00008AED}, {0x0000FABF, 0x00008B39}, {0x0000FAC0, 0x00008B8A},+{0x0000FAC1, 0x00008D08}, {0x0000FAC2, 0x00008F38}, {0x0000FAC3, 0x00009072}, {0x0000FAC4, 0x00009199},+{0x0000FAC5, 0x00009276}, {0x0000FAC6, 0x0000967C}, {0x0000FAC7, 0x000096E3}, {0x0000FAC8, 0x00009756},+{0x0000FAC9, 0x000097DB}, {0x0000FACA, 0x000097FF}, {0x0000FACB, 0x0000980B}, {0x0000FACC, 0x0000983B},+{0x0000FACD, 0x00009B12}, {0x0000FACE, 0x00009F9C}, {0x0000FACF, 0x0002284A}, {0x0000FAD0, 0x00022844},+{0x0000FAD1, 0x000233D5}, {0x0000FAD2, 0x00003B9D}, {0x0000FAD3, 0x00004018}, {0x0000FAD4, 0x00004039},+{0x0000FAD5, 0x00025249}, {0x0000FAD6, 0x00025CD0}, {0x0000FAD7, 0x00027ED3}, {0x0000FAD8, 0x00009F43},+{0x0000FAD9, 0x00009F8E}, {0x0000FB1D, 0x000005D9}, {0x0000FB1D, 0x000005B4}, {0x0000FB1F, 0x000005F2},+{0x0000FB1F, 0x000005B7}, {0x0000FB2A, 0x000005E9}, {0x0000FB2A, 0x000005C1}, {0x0000FB2B, 0x000005E9},+{0x0000FB2B, 0x000005C2}, {0x0000FB2C, 0x000005E9}, {0x0000FB2C, 0x000005BC}, {0x0000FB2C, 0x000005C1},+{0x0000FB2D, 0x000005E9}, {0x0000FB2D, 0x000005BC}, {0x0000FB2D, 0x000005C2}, {0x0000FB2E, 0x000005D0},+{0x0000FB2E, 0x000005B7}, {0x0000FB2F, 0x000005D0}, {0x0000FB2F, 0x000005B8}, {0x0000FB30, 0x000005D0},+{0x0000FB30, 0x000005BC}, {0x0000FB31, 0x000005D1}, {0x0000FB31, 0x000005BC}, {0x0000FB32, 0x000005D2},+{0x0000FB32, 0x000005BC}, {0x0000FB33, 0x000005D3}, {0x0000FB33, 0x000005BC}, {0x0000FB34, 0x000005D4},+{0x0000FB34, 0x000005BC}, {0x0000FB35, 0x000005D5}, {0x0000FB35, 0x000005BC}, {0x0000FB36, 0x000005D6},+{0x0000FB36, 0x000005BC}, {0x0000FB38, 0x000005D8}, {0x0000FB38, 0x000005BC}, {0x0000FB39, 0x000005D9},+{0x0000FB39, 0x000005BC}, {0x0000FB3A, 0x000005DA}, {0x0000FB3A, 0x000005BC}, {0x0000FB3B, 0x000005DB},+{0x0000FB3B, 0x000005BC}, {0x0000FB3C, 0x000005DC}, {0x0000FB3C, 0x000005BC}, {0x0000FB3E, 0x000005DE},+{0x0000FB3E, 0x000005BC}, {0x0000FB40, 0x000005E0}, {0x0000FB40, 0x000005BC}, {0x0000FB41, 0x000005E1},+{0x0000FB41, 0x000005BC}, {0x0000FB43, 0x000005E3}, {0x0000FB43, 0x000005BC}, {0x0000FB44, 0x000005E4},+{0x0000FB44, 0x000005BC}, {0x0000FB46, 0x000005E6}, {0x0000FB46, 0x000005BC}, {0x0000FB47, 0x000005E7},+{0x0000FB47, 0x000005BC}, {0x0000FB48, 0x000005E8}, {0x0000FB48, 0x000005BC}, {0x0000FB49, 0x000005E9},+{0x0000FB49, 0x000005BC}, {0x0000FB4A, 0x000005EA}, {0x0000FB4A, 0x000005BC}, {0x0000FB4B, 0x000005D5},+{0x0000FB4B, 0x000005B9}, {0x0000FB4C, 0x000005D1}, {0x0000FB4C, 0x000005BF}, {0x0000FB4D, 0x000005DB},+{0x0000FB4D, 0x000005BF}, {0x0000FB4E, 0x000005E4}, {0x0000FB4E, 0x000005BF}, {0x0001109A, 0x00011099},+{0x0001109A, 0x000110BA}, {0x0001109C, 0x0001109B}, {0x0001109C, 0x000110BA}, {0x000110AB, 0x000110A5},+{0x000110AB, 0x000110BA}, {0x0001112E, 0x00011131}, {0x0001112E, 0x00011127}, {0x0001112F, 0x00011132},+{0x0001112F, 0x00011127}, {0x0001134B, 0x00011347}, {0x0001134B, 0x0001133E}, {0x0001134C, 0x00011347},+{0x0001134C, 0x00011357}, {0x000114BB, 0x000114B9}, {0x000114BB, 0x000114BA}, {0x000114BC, 0x000114B9},+{0x000114BC, 0x000114B0}, {0x000114BE, 0x000114B9}, {0x000114BE, 0x000114BD}, {0x000115BA, 0x000115B8},+{0x000115BA, 0x000115AF}, {0x000115BB, 0x000115B9}, {0x000115BB, 0x000115AF}, {0x0001D15E, 0x0001D157},+{0x0001D15E, 0x0001D165}, {0x0001D15F, 0x0001D158}, {0x0001D15F, 0x0001D165}, {0x0001D160, 0x0001D158},+{0x0001D160, 0x0001D165}, {0x0001D160, 0x0001D16E}, {0x0001D161, 0x0001D158}, {0x0001D161, 0x0001D165},+{0x0001D161, 0x0001D16F}, {0x0001D162, 0x0001D158}, {0x0001D162, 0x0001D165}, {0x0001D162, 0x0001D170},+{0x0001D163, 0x0001D158}, {0x0001D163, 0x0001D165}, {0x0001D163, 0x0001D171}, {0x0001D164, 0x0001D158},+{0x0001D164, 0x0001D165}, {0x0001D164, 0x0001D172}, {0x0001D1BB, 0x0001D1B9}, {0x0001D1BB, 0x0001D165},+{0x0001D1BC, 0x0001D1BA}, {0x0001D1BC, 0x0001D165}, {0x0001D1BD, 0x0001D1B9}, {0x0001D1BD, 0x0001D165},+{0x0001D1BD, 0x0001D16E}, {0x0001D1BE, 0x0001D1BA}, {0x0001D1BE, 0x0001D165}, {0x0001D1BE, 0x0001D16E},+{0x0001D1BF, 0x0001D1B9}, {0x0001D1BF, 0x0001D165}, {0x0001D1BF, 0x0001D16F}, {0x0001D1C0, 0x0001D1BA},+{0x0001D1C0, 0x0001D165}, {0x0001D1C0, 0x0001D16F}, {0x0002F800, 0x00004E3D}, {0x0002F801, 0x00004E38},+{0x0002F802, 0x00004E41}, {0x0002F803, 0x00020122}, {0x0002F804, 0x00004F60}, {0x0002F805, 0x00004FAE},+{0x0002F806, 0x00004FBB}, {0x0002F807, 0x00005002}, {0x0002F808, 0x0000507A}, {0x0002F809, 0x00005099},+{0x0002F80A, 0x000050E7}, {0x0002F80B, 0x000050CF}, {0x0002F80C, 0x0000349E}, {0x0002F80D, 0x0002063A},+{0x0002F80E, 0x0000514D}, {0x0002F80F, 0x00005154}, {0x0002F810, 0x00005164}, {0x0002F811, 0x00005177},+{0x0002F812, 0x0002051C}, {0x0002F813, 0x000034B9}, {0x0002F814, 0x00005167}, {0x0002F815, 0x0000518D},+{0x0002F816, 0x0002054B}, {0x0002F817, 0x00005197}, {0x0002F818, 0x000051A4}, {0x0002F819, 0x00004ECC},+{0x0002F81A, 0x000051AC}, {0x0002F81B, 0x000051B5}, {0x0002F81C, 0x000291DF}, {0x0002F81D, 0x000051F5},+{0x0002F81E, 0x00005203}, {0x0002F81F, 0x000034DF}, {0x0002F820, 0x0000523B}, {0x0002F821, 0x00005246},+{0x0002F822, 0x00005272}, {0x0002F823, 0x00005277}, {0x0002F824, 0x00003515}, {0x0002F825, 0x000052C7},+{0x0002F826, 0x000052C9}, {0x0002F827, 0x000052E4}, {0x0002F828, 0x000052FA}, {0x0002F829, 0x00005305},+{0x0002F82A, 0x00005306}, {0x0002F82B, 0x00005317}, {0x0002F82C, 0x00005349}, {0x0002F82D, 0x00005351},+{0x0002F82E, 0x0000535A}, {0x0002F82F, 0x00005373}, {0x0002F830, 0x0000537D}, {0x0002F831, 0x0000537F},+{0x0002F832, 0x0000537F}, {0x0002F833, 0x0000537F}, {0x0002F834, 0x00020A2C}, {0x0002F835, 0x00007070},+{0x0002F836, 0x000053CA}, {0x0002F837, 0x000053DF}, {0x0002F838, 0x00020B63}, {0x0002F839, 0x000053EB},+{0x0002F83A, 0x000053F1}, {0x0002F83B, 0x00005406}, {0x0002F83C, 0x0000549E}, {0x0002F83D, 0x00005438},+{0x0002F83E, 0x00005448}, {0x0002F83F, 0x00005468}, {0x0002F840, 0x000054A2}, {0x0002F841, 0x000054F6},+{0x0002F842, 0x00005510}, {0x0002F843, 0x00005553}, {0x0002F844, 0x00005563}, {0x0002F845, 0x00005584},+{0x0002F846, 0x00005584}, {0x0002F847, 0x00005599}, {0x0002F848, 0x000055AB}, {0x0002F849, 0x000055B3},+{0x0002F84A, 0x000055C2}, {0x0002F84B, 0x00005716}, {0x0002F84C, 0x00005606}, {0x0002F84D, 0x00005717},+{0x0002F84E, 0x00005651}, {0x0002F84F, 0x00005674}, {0x0002F850, 0x00005207}, {0x0002F851, 0x000058EE},+{0x0002F852, 0x000057CE}, {0x0002F853, 0x000057F4}, {0x0002F854, 0x0000580D}, {0x0002F855, 0x0000578B},+{0x0002F856, 0x00005832}, {0x0002F857, 0x00005831}, {0x0002F858, 0x000058AC}, {0x0002F859, 0x000214E4},+{0x0002F85A, 0x000058F2}, {0x0002F85B, 0x000058F7}, {0x0002F85C, 0x00005906}, {0x0002F85D, 0x0000591A},+{0x0002F85E, 0x00005922}, {0x0002F85F, 0x00005962}, {0x0002F860, 0x000216A8}, {0x0002F861, 0x000216EA},+{0x0002F862, 0x000059EC}, {0x0002F863, 0x00005A1B}, {0x0002F864, 0x00005A27}, {0x0002F865, 0x000059D8},+{0x0002F866, 0x00005A66}, {0x0002F867, 0x000036EE}, {0x0002F868, 0x000036FC}, {0x0002F869, 0x00005B08},+{0x0002F86A, 0x00005B3E}, {0x0002F86B, 0x00005B3E}, {0x0002F86C, 0x000219C8}, {0x0002F86D, 0x00005BC3},+{0x0002F86E, 0x00005BD8}, {0x0002F86F, 0x00005BE7}, {0x0002F870, 0x00005BF3}, {0x0002F871, 0x00021B18},+{0x0002F872, 0x00005BFF}, {0x0002F873, 0x00005C06}, {0x0002F874, 0x00005F53}, {0x0002F875, 0x00005C22},+{0x0002F876, 0x00003781}, {0x0002F877, 0x00005C60}, {0x0002F878, 0x00005C6E}, {0x0002F879, 0x00005CC0},+{0x0002F87A, 0x00005C8D}, {0x0002F87B, 0x00021DE4}, {0x0002F87C, 0x00005D43}, {0x0002F87D, 0x00021DE6},+{0x0002F87E, 0x00005D6E}, {0x0002F87F, 0x00005D6B}, {0x0002F880, 0x00005D7C}, {0x0002F881, 0x00005DE1},+{0x0002F882, 0x00005DE2}, {0x0002F883, 0x0000382F}, {0x0002F884, 0x00005DFD}, {0x0002F885, 0x00005E28},+{0x0002F886, 0x00005E3D}, {0x0002F887, 0x00005E69}, {0x0002F888, 0x00003862}, {0x0002F889, 0x00022183},+{0x0002F88A, 0x0000387C}, {0x0002F88B, 0x00005EB0}, {0x0002F88C, 0x00005EB3}, {0x0002F88D, 0x00005EB6},+{0x0002F88E, 0x00005ECA}, {0x0002F88F, 0x0002A392}, {0x0002F890, 0x00005EFE}, {0x0002F891, 0x00022331},+{0x0002F892, 0x00022331}, {0x0002F893, 0x00008201}, {0x0002F894, 0x00005F22}, {0x0002F895, 0x00005F22},+{0x0002F896, 0x000038C7}, {0x0002F897, 0x000232B8}, {0x0002F898, 0x000261DA}, {0x0002F899, 0x00005F62},+{0x0002F89A, 0x00005F6B}, {0x0002F89B, 0x000038E3}, {0x0002F89C, 0x00005F9A}, {0x0002F89D, 0x00005FCD},+{0x0002F89E, 0x00005FD7}, {0x0002F89F, 0x00005FF9}, {0x0002F8A0, 0x00006081}, {0x0002F8A1, 0x0000393A},+{0x0002F8A2, 0x0000391C}, {0x0002F8A3, 0x00006094}, {0x0002F8A4, 0x000226D4}, {0x0002F8A5, 0x000060C7},+{0x0002F8A6, 0x00006148}, {0x0002F8A7, 0x0000614C}, {0x0002F8A8, 0x0000614E}, {0x0002F8A9, 0x0000614C},+{0x0002F8AA, 0x0000617A}, {0x0002F8AB, 0x0000618E}, {0x0002F8AC, 0x000061B2}, {0x0002F8AD, 0x000061A4},+{0x0002F8AE, 0x000061AF}, {0x0002F8AF, 0x000061DE}, {0x0002F8B0, 0x000061F2}, {0x0002F8B1, 0x000061F6},+{0x0002F8B2, 0x00006210}, {0x0002F8B3, 0x0000621B}, {0x0002F8B4, 0x0000625D}, {0x0002F8B5, 0x000062B1},+{0x0002F8B6, 0x000062D4}, {0x0002F8B7, 0x00006350}, {0x0002F8B8, 0x00022B0C}, {0x0002F8B9, 0x0000633D},+{0x0002F8BA, 0x000062FC}, {0x0002F8BB, 0x00006368}, {0x0002F8BC, 0x00006383}, {0x0002F8BD, 0x000063E4},+{0x0002F8BE, 0x00022BF1}, {0x0002F8BF, 0x00006422}, {0x0002F8C0, 0x000063C5}, {0x0002F8C1, 0x000063A9},+{0x0002F8C2, 0x00003A2E}, {0x0002F8C3, 0x00006469}, {0x0002F8C4, 0x0000647E}, {0x0002F8C5, 0x0000649D},+{0x0002F8C6, 0x00006477}, {0x0002F8C7, 0x00003A6C}, {0x0002F8C8, 0x0000654F}, {0x0002F8C9, 0x0000656C},+{0x0002F8CA, 0x0002300A}, {0x0002F8CB, 0x000065E3}, {0x0002F8CC, 0x000066F8}, {0x0002F8CD, 0x00006649},+{0x0002F8CE, 0x00003B19}, {0x0002F8CF, 0x00006691}, {0x0002F8D0, 0x00003B08}, {0x0002F8D1, 0x00003AE4},+{0x0002F8D2, 0x00005192}, {0x0002F8D3, 0x00005195}, {0x0002F8D4, 0x00006700}, {0x0002F8D5, 0x0000669C},+{0x0002F8D6, 0x000080AD}, {0x0002F8D7, 0x000043D9}, {0x0002F8D8, 0x00006717}, {0x0002F8D9, 0x0000671B},+{0x0002F8DA, 0x00006721}, {0x0002F8DB, 0x0000675E}, {0x0002F8DC, 0x00006753}, {0x0002F8DD, 0x000233C3},+{0x0002F8DE, 0x00003B49}, {0x0002F8DF, 0x000067FA}, {0x0002F8E0, 0x00006785}, {0x0002F8E1, 0x00006852},+{0x0002F8E2, 0x00006885}, {0x0002F8E3, 0x0002346D}, {0x0002F8E4, 0x0000688E}, {0x0002F8E5, 0x0000681F},+{0x0002F8E6, 0x00006914}, {0x0002F8E7, 0x00003B9D}, {0x0002F8E8, 0x00006942}, {0x0002F8E9, 0x000069A3},+{0x0002F8EA, 0x000069EA}, {0x0002F8EB, 0x00006AA8}, {0x0002F8EC, 0x000236A3}, {0x0002F8ED, 0x00006ADB},+{0x0002F8EE, 0x00003C18}, {0x0002F8EF, 0x00006B21}, {0x0002F8F0, 0x000238A7}, {0x0002F8F1, 0x00006B54},+{0x0002F8F2, 0x00003C4E}, {0x0002F8F3, 0x00006B72}, {0x0002F8F4, 0x00006B9F}, {0x0002F8F5, 0x00006BBA},+{0x0002F8F6, 0x00006BBB}, {0x0002F8F7, 0x00023A8D}, {0x0002F8F8, 0x00021D0B}, {0x0002F8F9, 0x00023AFA},+{0x0002F8FA, 0x00006C4E}, {0x0002F8FB, 0x00023CBC}, {0x0002F8FC, 0x00006CBF}, {0x0002F8FD, 0x00006CCD},+{0x0002F8FE, 0x00006C67}, {0x0002F8FF, 0x00006D16}, {0x0002F900, 0x00006D3E}, {0x0002F901, 0x00006D77},+{0x0002F902, 0x00006D41}, {0x0002F903, 0x00006D69}, {0x0002F904, 0x00006D78}, {0x0002F905, 0x00006D85},+{0x0002F906, 0x00023D1E}, {0x0002F907, 0x00006D34}, {0x0002F908, 0x00006E2F}, {0x0002F909, 0x00006E6E},+{0x0002F90A, 0x00003D33}, {0x0002F90B, 0x00006ECB}, {0x0002F90C, 0x00006EC7}, {0x0002F90D, 0x00023ED1},+{0x0002F90E, 0x00006DF9}, {0x0002F90F, 0x00006F6E}, {0x0002F910, 0x00023F5E}, {0x0002F911, 0x00023F8E},+{0x0002F912, 0x00006FC6}, {0x0002F913, 0x00007039}, {0x0002F914, 0x0000701E}, {0x0002F915, 0x0000701B},+{0x0002F916, 0x00003D96}, {0x0002F917, 0x0000704A}, {0x0002F918, 0x0000707D}, {0x0002F919, 0x00007077},+{0x0002F91A, 0x000070AD}, {0x0002F91B, 0x00020525}, {0x0002F91C, 0x00007145}, {0x0002F91D, 0x00024263},+{0x0002F91E, 0x0000719C}, {0x0002F91F, 0x000243AB}, {0x0002F920, 0x00007228}, {0x0002F921, 0x00007235},+{0x0002F922, 0x00007250}, {0x0002F923, 0x00024608}, {0x0002F924, 0x00007280}, {0x0002F925, 0x00007295},+{0x0002F926, 0x00024735}, {0x0002F927, 0x00024814}, {0x0002F928, 0x0000737A}, {0x0002F929, 0x0000738B},+{0x0002F92A, 0x00003EAC}, {0x0002F92B, 0x000073A5}, {0x0002F92C, 0x00003EB8}, {0x0002F92D, 0x00003EB8},+{0x0002F92E, 0x00007447}, {0x0002F92F, 0x0000745C}, {0x0002F930, 0x00007471}, {0x0002F931, 0x00007485},+{0x0002F932, 0x000074CA}, {0x0002F933, 0x00003F1B}, {0x0002F934, 0x00007524}, {0x0002F935, 0x00024C36},+{0x0002F936, 0x0000753E}, {0x0002F937, 0x00024C92}, {0x0002F938, 0x00007570}, {0x0002F939, 0x0002219F},+{0x0002F93A, 0x00007610}, {0x0002F93B, 0x00024FA1}, {0x0002F93C, 0x00024FB8}, {0x0002F93D, 0x00025044},+{0x0002F93E, 0x00003FFC}, {0x0002F93F, 0x00004008}, {0x0002F940, 0x000076F4}, {0x0002F941, 0x000250F3},+{0x0002F942, 0x000250F2}, {0x0002F943, 0x00025119}, {0x0002F944, 0x00025133}, {0x0002F945, 0x0000771E},+{0x0002F946, 0x0000771F}, {0x0002F947, 0x0000771F}, {0x0002F948, 0x0000774A}, {0x0002F949, 0x00004039},+{0x0002F94A, 0x0000778B}, {0x0002F94B, 0x00004046}, {0x0002F94C, 0x00004096}, {0x0002F94D, 0x0002541D},+{0x0002F94E, 0x0000784E}, {0x0002F94F, 0x0000788C}, {0x0002F950, 0x000078CC}, {0x0002F951, 0x000040E3},+{0x0002F952, 0x00025626}, {0x0002F953, 0x00007956}, {0x0002F954, 0x0002569A}, {0x0002F955, 0x000256C5},+{0x0002F956, 0x0000798F}, {0x0002F957, 0x000079EB}, {0x0002F958, 0x0000412F}, {0x0002F959, 0x00007A40},+{0x0002F95A, 0x00007A4A}, {0x0002F95B, 0x00007A4F}, {0x0002F95C, 0x0002597C}, {0x0002F95D, 0x00025AA7},+{0x0002F95E, 0x00025AA7}, {0x0002F95F, 0x00007AEE}, {0x0002F960, 0x00004202}, {0x0002F961, 0x00025BAB},+{0x0002F962, 0x00007BC6}, {0x0002F963, 0x00007BC9}, {0x0002F964, 0x00004227}, {0x0002F965, 0x00025C80},+{0x0002F966, 0x00007CD2}, {0x0002F967, 0x000042A0}, {0x0002F968, 0x00007CE8}, {0x0002F969, 0x00007CE3},+{0x0002F96A, 0x00007D00}, {0x0002F96B, 0x00025F86}, {0x0002F96C, 0x00007D63}, {0x0002F96D, 0x00004301},+{0x0002F96E, 0x00007DC7}, {0x0002F96F, 0x00007E02}, {0x0002F970, 0x00007E45}, {0x0002F971, 0x00004334},+{0x0002F972, 0x00026228}, {0x0002F973, 0x00026247}, {0x0002F974, 0x00004359}, {0x0002F975, 0x000262D9},+{0x0002F976, 0x00007F7A}, {0x0002F977, 0x0002633E}, {0x0002F978, 0x00007F95}, {0x0002F979, 0x00007FFA},+{0x0002F97A, 0x00008005}, {0x0002F97B, 0x000264DA}, {0x0002F97C, 0x00026523}, {0x0002F97D, 0x00008060},+{0x0002F97E, 0x000265A8}, {0x0002F97F, 0x00008070}, {0x0002F980, 0x0002335F}, {0x0002F981, 0x000043D5},+{0x0002F982, 0x000080B2}, {0x0002F983, 0x00008103}, {0x0002F984, 0x0000440B}, {0x0002F985, 0x0000813E},+{0x0002F986, 0x00005AB5}, {0x0002F987, 0x000267A7}, {0x0002F988, 0x000267B5}, {0x0002F989, 0x00023393},+{0x0002F98A, 0x0002339C}, {0x0002F98B, 0x00008201}, {0x0002F98C, 0x00008204}, {0x0002F98D, 0x00008F9E},+{0x0002F98E, 0x0000446B}, {0x0002F98F, 0x00008291}, {0x0002F990, 0x0000828B}, {0x0002F991, 0x0000829D},+{0x0002F992, 0x000052B3}, {0x0002F993, 0x000082B1}, {0x0002F994, 0x000082B3}, {0x0002F995, 0x000082BD},+{0x0002F996, 0x000082E6}, {0x0002F997, 0x00026B3C}, {0x0002F998, 0x000082E5}, {0x0002F999, 0x0000831D},+{0x0002F99A, 0x00008363}, {0x0002F99B, 0x000083AD}, {0x0002F99C, 0x00008323}, {0x0002F99D, 0x000083BD},+{0x0002F99E, 0x000083E7}, {0x0002F99F, 0x00008457}, {0x0002F9A0, 0x00008353}, {0x0002F9A1, 0x000083CA},+{0x0002F9A2, 0x000083CC}, {0x0002F9A3, 0x000083DC}, {0x0002F9A4, 0x00026C36}, {0x0002F9A5, 0x00026D6B},+{0x0002F9A6, 0x00026CD5}, {0x0002F9A7, 0x0000452B}, {0x0002F9A8, 0x000084F1}, {0x0002F9A9, 0x000084F3},+{0x0002F9AA, 0x00008516}, {0x0002F9AB, 0x000273CA}, {0x0002F9AC, 0x00008564}, {0x0002F9AD, 0x00026F2C},+{0x0002F9AE, 0x0000455D}, {0x0002F9AF, 0x00004561}, {0x0002F9B0, 0x00026FB1}, {0x0002F9B1, 0x000270D2},+{0x0002F9B2, 0x0000456B}, {0x0002F9B3, 0x00008650}, {0x0002F9B4, 0x0000865C}, {0x0002F9B5, 0x00008667},+{0x0002F9B6, 0x00008669}, {0x0002F9B7, 0x000086A9}, {0x0002F9B8, 0x00008688}, {0x0002F9B9, 0x0000870E},+{0x0002F9BA, 0x000086E2}, {0x0002F9BB, 0x00008779}, {0x0002F9BC, 0x00008728}, {0x0002F9BD, 0x0000876B},+{0x0002F9BE, 0x00008786}, {0x0002F9BF, 0x000045D7}, {0x0002F9C0, 0x000087E1}, {0x0002F9C1, 0x00008801},+{0x0002F9C2, 0x000045F9}, {0x0002F9C3, 0x00008860}, {0x0002F9C4, 0x00008863}, {0x0002F9C5, 0x00027667},+{0x0002F9C6, 0x000088D7}, {0x0002F9C7, 0x000088DE}, {0x0002F9C8, 0x00004635}, {0x0002F9C9, 0x000088FA},+{0x0002F9CA, 0x000034BB}, {0x0002F9CB, 0x000278AE}, {0x0002F9CC, 0x00027966}, {0x0002F9CD, 0x000046BE},+{0x0002F9CE, 0x000046C7}, {0x0002F9CF, 0x00008AA0}, {0x0002F9D0, 0x00008AED}, {0x0002F9D1, 0x00008B8A},+{0x0002F9D2, 0x00008C55}, {0x0002F9D3, 0x00027CA8}, {0x0002F9D4, 0x00008CAB}, {0x0002F9D5, 0x00008CC1},+{0x0002F9D6, 0x00008D1B}, {0x0002F9D7, 0x00008D77}, {0x0002F9D8, 0x00027F2F}, {0x0002F9D9, 0x00020804},+{0x0002F9DA, 0x00008DCB}, {0x0002F9DB, 0x00008DBC}, {0x0002F9DC, 0x00008DF0}, {0x0002F9DD, 0x000208DE},+{0x0002F9DE, 0x00008ED4}, {0x0002F9DF, 0x00008F38}, {0x0002F9E0, 0x000285D2}, {0x0002F9E1, 0x000285ED},+{0x0002F9E2, 0x00009094}, {0x0002F9E3, 0x000090F1}, {0x0002F9E4, 0x00009111}, {0x0002F9E5, 0x0002872E},+{0x0002F9E6, 0x0000911B}, {0x0002F9E7, 0x00009238}, {0x0002F9E8, 0x000092D7}, {0x0002F9E9, 0x000092D8},+{0x0002F9EA, 0x0000927C}, {0x0002F9EB, 0x000093F9}, {0x0002F9EC, 0x00009415}, {0x0002F9ED, 0x00028BFA},+{0x0002F9EE, 0x0000958B}, {0x0002F9EF, 0x00004995}, {0x0002F9F0, 0x000095B7}, {0x0002F9F1, 0x00028D77},+{0x0002F9F2, 0x000049E6}, {0x0002F9F3, 0x000096C3}, {0x0002F9F4, 0x00005DB2}, {0x0002F9F5, 0x00009723},+{0x0002F9F6, 0x00029145}, {0x0002F9F7, 0x0002921A}, {0x0002F9F8, 0x00004A6E}, {0x0002F9F9, 0x00004A76},+{0x0002F9FA, 0x000097E0}, {0x0002F9FB, 0x0002940A}, {0x0002F9FC, 0x00004AB2}, {0x0002F9FD, 0x00029496},+{0x0002F9FE, 0x0000980B}, {0x0002F9FF, 0x0000980B}, {0x0002FA00, 0x00009829}, {0x0002FA01, 0x000295B6},+{0x0002FA02, 0x000098E2}, {0x0002FA03, 0x00004B33}, {0x0002FA04, 0x00009929}, {0x0002FA05, 0x000099A7},+{0x0002FA06, 0x000099C2}, {0x0002FA07, 0x000099FE}, {0x0002FA08, 0x00004BCE}, {0x0002FA09, 0x00029B30},+{0x0002FA0A, 0x00009B12}, {0x0002FA0B, 0x00009C40}, {0x0002FA0C, 0x00009CFD}, {0x0002FA0D, 0x00004CCE},+{0x0002FA0E, 0x00004CED}, {0x0002FA0F, 0x00009D67}, {0x0002FA10, 0x0002A0CE}, {0x0002FA11, 0x00004CF8},+{0x0002FA12, 0x0002A105}, {0x0002FA13, 0x0002A20E}, {0x0002FA14, 0x0002A291}, {0x0002FA15, 0x00009EBB},+{0x0002FA16, 0x00004D56}, {0x0002FA17, 0x00009EF9}, {0x0002FA18, 0x00009EFE}, {0x0002FA19, 0x00009F05},+{0x0002FA1A, 0x00009F0F}, {0x0002FA1B, 0x00009F16}, {0x0002FA1D, 0x0002A600},+};++static std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {+ std::string result;+ for (size_t i = 0; i < cps.size(); ++i) {+ result.append(unicode_cpt_to_utf8(cps[i]));+ }+ return result;+}++static uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {+ assert(offset < utf8.size());+ if (!(utf8[offset + 0] & 0x80)) {+ auto result = utf8[offset + 0];+ offset += 1;+ return result;+ }+ if (!(utf8[offset + 0] & 0x40)) {+ throw std::invalid_argument("invalid character");+ }+ if (!(utf8[offset + 0] & 0x20)) {+ if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) {+ throw std::invalid_argument("invalid character");+ }+ auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);+ offset += 2;+ return result;+ }+ if (!(utf8[offset + 0] & 0x10)) {+ if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) {+ throw std::invalid_argument("invalid character");+ }+ auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);+ offset += 3;+ return result;+ }+ if (!(utf8[offset + 0] & 0x08)) {+ if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) {+ throw std::invalid_argument("invalid character");+ }+ auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);+ offset += 4;+ return result;+ }+ throw std::invalid_argument("invalid string");+}++static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {+ std::vector<uint16_t> result;+ if (/* 0x0000 <= cp && */ cp <= 0xffff) {+ result.emplace_back(cp);+ }+ else if (0x10000 <= cp && cp <= 0x10ffff) {+ result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));+ result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));+ }+ else {+ throw std::invalid_argument("invalid cpt");+ }+ return result;+}++//static std::vector<uint16_t> unicode_cpts_to_utf16(const std::vector<uint32_t> & cps) {+// std::vector<uint16_t> result;+// for (size_t i = 0; i < cps.size(); ++i) {+// auto temp = unicode_cpt_to_utf16(cps[i]);+// result.insert(result.end(), temp.begin(), temp.end());+// }+// return result;+//}++static uint32_t cpt_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {+ assert(offset < utf16.size());+ if (((utf16[0] >> 10) << 10) != 0xd800) {+ auto result = utf16[offset + 0];+ offset += 1;+ return result;+ }++ if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) {+ throw std::invalid_argument("invalid character");+ }++ auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));+ offset += 2;+ return result;+}++//static std::vector<uint32_t> unicode_cpts_from_utf16(const std::vector<uint16_t> & utf16) {+// std::vector<uint32_t> result;+// size_t offset = 0;+// while (offset < utf16.size()) {+// result.push_back(cpt_from_utf16(utf16, offset));+// }+// return result;+//}++static std::unordered_map<uint32_t, int> unicode_cpt_type_map() {+ std::unordered_map<uint32_t, int> cpt_types;+ for (auto p : unicode_ranges_digit) {+ for (auto i = p.first; i <= p.second; ++ i) {+ cpt_types[i] = CODEPOINT_TYPE_DIGIT;+ }+ }+ for (auto p : unicode_ranges_letter) {+ for (auto i = p.first; i <= p.second; ++ i) {+ cpt_types[i] = CODEPOINT_TYPE_LETTER;+ }+ }+ for (auto p : unicode_ranges_whitespace) {+ for (auto i = p.first; i <= p.second; ++ i) {+ cpt_types[i] = CODEPOINT_TYPE_WHITESPACE;+ }+ }+ for (auto p : unicode_ranges_accent_mark) {+ for (auto i = p.first; i <= p.second; ++ i) {+ cpt_types[i] = CODEPOINT_TYPE_ACCENT_MARK;+ }+ }+ for (auto p : unicode_ranges_punctuation) {+ for (auto i = p.first; i <= p.second; ++ i) {+ cpt_types[i] = CODEPOINT_TYPE_PUNCTUATION;+ }+ }+ for (auto p : unicode_ranges_symbol) {+ for (auto i = p.first; i <= p.second; ++i) {+ cpt_types[i] = CODEPOINT_TYPE_SYMBOL;+ }+ }+ for (auto p : unicode_ranges_control) {+ for (auto i = p.first; i <= p.second; ++ i) {+ cpt_types[i] = CODEPOINT_TYPE_CONTROL;+ }+ }+ return cpt_types;+}++static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {+ std::unordered_map<uint8_t, std::string> map;+ for (int ch = u'!'; ch <= u'~'; ++ch) {+ assert(0 <= ch && ch < 256);+ map[ch] = unicode_cpt_to_utf8(ch);+ }+ for (int ch = u'¡'; ch <= u'¬'; ++ch) {+ assert(0 <= ch && ch < 256);+ map[ch] = unicode_cpt_to_utf8(ch);+ }+ for (int ch = u'®'; ch <= u'ÿ'; ++ch) {+ assert(0 <= ch && ch < 256);+ map[ch] = unicode_cpt_to_utf8(ch);+ }+ auto n = 0;+ for (int ch = 0; ch < 256; ++ch) {+ if (map.find(ch) == map.end()) {+ map[ch] = unicode_cpt_to_utf8(256 + n);+ ++n;+ }+ }+ return map;+}++static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {+ std::unordered_map<std::string, uint8_t> map;+ for (int ch = u'!'; ch <= u'~'; ++ch) {+ assert(0 <= ch && ch < 256);+ map[unicode_cpt_to_utf8(ch)] = ch;+ }+ for (int ch = u'¡'; ch <= u'¬'; ++ch) {+ assert(0 <= ch && ch < 256);+ map[unicode_cpt_to_utf8(ch)] = ch;+ }+ for (int ch = u'®'; ch <= u'ÿ'; ++ch) {+ assert(0 <= ch && ch < 256);+ map[unicode_cpt_to_utf8(ch)] = ch;+ }+ auto n = 0;+ for (int ch = 0; ch < 256; ++ch) {+ if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) {+ map[unicode_cpt_to_utf8(256 + n)] = ch;+ ++n;+ }+ }+ return map;+}++//+// interface+//++std::string unicode_cpt_to_utf8(uint32_t cp) {+ std::string result;+ if (/* 0x00 <= cp && */ cp <= 0x7f) {+ result.push_back(cp);+ }+ else if (0x80 <= cp && cp <= 0x7ff) {+ result.push_back(0xc0 | ((cp >> 6) & 0x1f));+ result.push_back(0x80 | (cp & 0x3f));+ }+ else if (0x800 <= cp && cp <= 0xffff) {+ result.push_back(0xe0 | ((cp >> 12) & 0x0f));+ result.push_back(0x80 | ((cp >> 6) & 0x3f));+ result.push_back(0x80 | (cp & 0x3f));+ }+ else if (0x10000 <= cp && cp <= 0x10ffff) {+ result.push_back(0xf0 | ((cp >> 18) & 0x07));+ result.push_back(0x80 | ((cp >> 12) & 0x3f));+ result.push_back(0x80 | ((cp >> 6) & 0x3f));+ result.push_back(0x80 | (cp & 0x3f));+ }+ else {+ throw std::invalid_argument("invalid codepoint");+ }+ return result;+}++std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {+ std::vector<uint32_t> result;+ result.reserve(cpts.size());+ for (size_t i = 0; i < cpts.size(); ++i) {+ auto it = unicode_map_nfd.find(cpts[i]);+ if (it == unicode_map_nfd.end()) {+ result.push_back(cpts[i]);+ } else {+ result.push_back(it->second);+ }+ }+ return result;+}++std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {+ std::vector<uint32_t> result;+ size_t offset = 0;+ while (offset < utf8.size()) {+ result.push_back(unicode_cpt_from_utf8(utf8, offset));+ }+ return result;+}++int unicode_cpt_type(uint32_t cp) {+ static std::unordered_map<uint32_t, int> cpt_types = unicode_cpt_type_map();+ const auto it = cpt_types.find(cp);+ return it == cpt_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : it->second;+}++int unicode_cpt_type(const std::string & utf8) {+ if (utf8.length() == 0) {+ return CODEPOINT_TYPE_UNIDENTIFIED;+ }+ size_t offset = 0;+ return unicode_cpt_type(unicode_cpt_from_utf8(utf8, offset));+}++std::string unicode_byte_to_utf8(uint8_t byte) {+ static std::unordered_map<uint8_t, std::string> map = unicode_byte_to_utf8_map();+ return map.at(byte);+}++uint8_t unicode_utf8_to_byte(const std::string & utf8) {+ static std::unordered_map<std::string, uint8_t> map = unicode_utf8_to_byte_map();+ return map.at(utf8);+}+diff --git a/llama.cpp/unicode.h b/llama.cpp/unicode.hindex aeca879e..6d14a5a3 100644--- a/llama.cpp/unicode.h+++ b/llama.cpp/unicode.h@@ -1,462 +1,26 @@-#pragma once+#pragma once-#include <cassert>-#include <stdexcept>+#include <cstdint>+#include <string>#include <vector>-#include <unordered_map>--static const std::vector<std::pair<uint32_t, uint32_t>> digit_ranges = {-{0x30, 0x39}, {0xB2, 0xB3}, {0xB9, 0xB9}, {0x660, 0x669}, {0x6F0, 0x6F9}, {0x7C0, 0x7C9}, {0x966, 0x96F}, {0x9E6, 0x9EF}, {0xA66, 0xA6F}, {0xAE6, 0xAEF}, {0xB66, 0xB6F}, {0xBE6, 0xBEF}, {0xC66, 0xC6F},-{0xCE6, 0xCEF}, {0xD66, 0xD6F}, {0xDE6, 0xDEF}, {0xE50, 0xE59}, {0xED0, 0xED9}, {0xF20, 0xF29}, {0x1040, 0x1049}, {0x1090, 0x1099}, {0x1369, 0x1371}, {0x17E0, 0x17E9}, {0x1810, 0x1819}, {0x1946, 0x194F},-{0x19D0, 0x19DA}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1B50, 0x1B59}, {0x1BB0, 0x1BB9}, {0x1C40, 0x1C49}, {0x1C50, 0x1C59}, {0x2070, 0x2070}, {0x2074, 0x2079}, {0x2080, 0x2089}, {0x2460, 0x2468},-{0x2474, 0x247C}, {0x2488, 0x2490}, {0x24EA, 0x24EA}, {0x24F5, 0x24FD}, {0x24FF, 0x24FF}, {0x2776, 0x277E}, {0x2780, 0x2788}, {0x278A, 0x2792}, {0xA620, 0xA629}, {0xA8D0, 0xA8D9}, {0xA900, 0xA909},-{0xA9D0, 0xA9D9}, {0xA9F0, 0xA9F9}, {0xAA50, 0xAA59}, {0xABF0, 0xABF9}, {0xFF10, 0xFF19}, {0x104A0, 0x104A9}, {0x10A40, 0x10A43}, {0x10D30, 0x10D39}, {0x10E60, 0x10E68}, {0x11052, 0x1105A},-{0x11066, 0x1106F}, {0x110F0, 0x110F9}, {0x11136, 0x1113F}, {0x111D0, 0x111D9}, {0x112F0, 0x112F9}, {0x11450, 0x11459}, {0x114D0, 0x114D9}, {0x11650, 0x11659}, {0x116C0, 0x116C9}, {0x11730, 0x11739},-{0x118E0, 0x118E9}, {0x11950, 0x11959}, {0x11C50, 0x11C59}, {0x11D50, 0x11D59}, {0x11DA0, 0x11DA9}, {0x16A60, 0x16A69}, {0x16B50, 0x16B59}, {0x1D7CE, 0x1D7FF}, {0x1E140, 0x1E149}, {0x1E2F0, 0x1E2F9},-{0x1E950, 0x1E959}, {0x1F100, 0x1F10A}, {0x1FBF0, 0x1FBF9},-};--static const std::vector<std::pair<uint32_t, uint32_t>> letter_ranges = {-{0x41, 0x5A}, {0x61, 0x7A}, {0xAA, 0xAA}, {0xB5, 0xB5}, {0xBA, 0xBA}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374},-{0x376, 0x377}, {0x37A, 0x37D}, {0x37F, 0x37F}, {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, {0x559, 0x559},-{0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, {0x620, 0x64A}, {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6EF}, {0x6FA, 0x6FC}, {0x6FF, 0x6FF}, {0x710, 0x710},-{0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, {0x7CA, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A},-{0x8A0, 0x8B4}, {0x8B6, 0x8C7}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2},-{0x9B6, 0x9B9}, {0x9BD, 0x9BD}, {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9F0, 0x9F1}, {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, {0xA32, 0xA33},-{0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, {0xA93, 0xAA8}, {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, {0xABD, 0xABD},-{0xAD0, 0xAD0}, {0xAE0, 0xAE1}, {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, {0xB13, 0xB28}, {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, {0xB5C, 0xB5D}, {0xB5F, 0xB61},-{0xB71, 0xB71}, {0xB83, 0xB83}, {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0},-{0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC60, 0xC61}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, {0xCAA, 0xCB3},-{0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDE, 0xCDE}, {0xCE0, 0xCE1}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD5F, 0xD61},-{0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, {0xDC0, 0xDC6}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A},-{0xE8C, 0xEA3}, {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, {0xEC0, 0xEC4}, {0xEC6, 0xEC6}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C},-{0x1000, 0x102A}, {0x103F, 0x103F}, {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, {0x1075, 0x1081}, {0x108E, 0x108E}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7},-{0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5},-{0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1380, 0x138F}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C},-{0x166F, 0x167F}, {0x1681, 0x169A}, {0x16A0, 0x16EA}, {0x16F1, 0x16F8}, {0x1700, 0x170C}, {0x170E, 0x1711}, {0x1720, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176C}, {0x176E, 0x1770}, {0x1780, 0x17B3},-{0x17D7, 0x17D7}, {0x17DC, 0x17DC}, {0x1820, 0x1878}, {0x1880, 0x1884}, {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1950, 0x196D}, {0x1970, 0x1974}, {0x1980, 0x19AB},-{0x19B0, 0x19C9}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, {0x1B45, 0x1B4B}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BAF}, {0x1BBA, 0x1BE5}, {0x1C00, 0x1C23}, {0x1C4D, 0x1C4F},-{0x1C5A, 0x1C7D}, {0x1C80, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, {0x1F18, 0x1F1D},-{0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4},-{0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2071, 0x2071}, {0x207F, 0x207F}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107},-{0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, {0x214E, 0x214E},-{0x2183, 0x2184}, {0x2C00, 0x2C2E}, {0x2C30, 0x2C5E}, {0x2C60, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F},-{0x2D80, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3006},-{0x3031, 0x3035}, {0x303B, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3400, 0x4DBF},-{0x4E00, 0x9FFC}, {0xA000, 0xA48C}, {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, {0xA610, 0xA61F}, {0xA62A, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6E5}, {0xA717, 0xA71F}, {0xA722, 0xA788},-{0xA78B, 0xA7BF}, {0xA7C2, 0xA7CA}, {0xA7F5, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, {0xA80C, 0xA822}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8F2, 0xA8F7}, {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE},-{0xA90A, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, {0xA984, 0xA9B2}, {0xA9CF, 0xA9CF}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9EF}, {0xA9FA, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, {0xAA44, 0xAA4B},-{0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4},-{0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB},-{0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, {0xFB43, 0xFB44},-{0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7},-{0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA},-{0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x10300, 0x1031F}, {0x1032D, 0x10340}, {0x10342, 0x10349}, {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, {0x10400, 0x1049D},-{0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835},-{0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10860, 0x10876}, {0x10880, 0x1089E}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x10900, 0x10915}, {0x10920, 0x10939}, {0x10980, 0x109B7},-{0x109BE, 0x109BF}, {0x10A00, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, {0x10A60, 0x10A7C}, {0x10A80, 0x10A9C}, {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, {0x10B00, 0x10B35},-{0x10B40, 0x10B55}, {0x10B60, 0x10B72}, {0x10B80, 0x10B91}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10D00, 0x10D23}, {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, {0x10F00, 0x10F1C},-{0x10F27, 0x10F27}, {0x10F30, 0x10F45}, {0x10FB0, 0x10FC4}, {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x11103, 0x11126}, {0x11144, 0x11144}, {0x11147, 0x11147},-{0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, {0x111DA, 0x111DA}, {0x111DC, 0x111DC}, {0x11200, 0x11211}, {0x11213, 0x1122B}, {0x11280, 0x11286}, {0x11288, 0x11288},-{0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339},-{0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x11580, 0x115AE},-{0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x11700, 0x1171A}, {0x11800, 0x1182B}, {0x118A0, 0x118DF}, {0x118FF, 0x11906}, {0x11909, 0x11909},-{0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00},-{0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AC0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C72, 0x11C8F},-{0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11EE0, 0x11EF2}, {0x11FB0, 0x11FB0},-{0x12000, 0x12399}, {0x12480, 0x12543}, {0x13000, 0x1342E}, {0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, {0x16B63, 0x16B77},-{0x16B7D, 0x16B8F}, {0x16E40, 0x16E7F}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08},-{0x1B000, 0x1B11E}, {0x1B150, 0x1B152}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C},-{0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514},-{0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA},-{0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D},-{0x1E14E, 0x1E14E}, {0x1E2C0, 0x1E2EB}, {0x1E800, 0x1E8C4}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27},-{0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52},-{0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72},-{0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x20000, 0x2A6DD}, {0x2A700, 0x2B734},-{0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A},-};--static const std::vector<std::pair<uint32_t, uint32_t>> whitespace_ranges = {-{0x9, 0xD}, {0x1C, 0x20}, {0x85, 0x85}, {0xA0, 0xA0}, {0x1680, 0x1680}, {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},-};--static const std::vector<std::pair<uint32_t, uint32_t>> accent_mark_ranges = {-{0x300, 0x36F}, {0x483, 0x489}, {0x591, 0x5BD}, {0x5BF, 0x5BF}, {0x5C1, 0x5C2}, {0x5C4, 0x5C5}, {0x5C7, 0x5C7}, {0x610, 0x61A}, {0x64B, 0x65F}, {0x670, 0x670}, {0x6D6, 0x6DC}, {0x6DF, 0x6E4},-{0x6E7, 0x6E8}, {0x6EA, 0x6ED}, {0x711, 0x711}, {0x730, 0x74A}, {0x7A6, 0x7B0}, {0x7EB, 0x7F3}, {0x7FD, 0x7FD}, {0x816, 0x819}, {0x81B, 0x823}, {0x825, 0x827}, {0x829, 0x82D}, {0x859, 0x85B},-{0x8D3, 0x8E1}, {0x8E3, 0x903}, {0x93A, 0x93C}, {0x93E, 0x94F}, {0x951, 0x957}, {0x962, 0x963}, {0x981, 0x983}, {0x9BC, 0x9BC}, {0x9BE, 0x9C4}, {0x9C7, 0x9C8}, {0x9CB, 0x9CD}, {0x9D7, 0x9D7},-{0x9E2, 0x9E3}, {0x9FE, 0x9FE}, {0xA01, 0xA03}, {0xA3C, 0xA3C}, {0xA3E, 0xA42}, {0xA47, 0xA48}, {0xA4B, 0xA4D}, {0xA51, 0xA51}, {0xA70, 0xA71}, {0xA75, 0xA75}, {0xA81, 0xA83}, {0xABC, 0xABC},-{0xABE, 0xAC5}, {0xAC7, 0xAC9}, {0xACB, 0xACD}, {0xAE2, 0xAE3}, {0xAFA, 0xAFF}, {0xB01, 0xB03}, {0xB3C, 0xB3C}, {0xB3E, 0xB44}, {0xB47, 0xB48}, {0xB4B, 0xB4D}, {0xB55, 0xB57}, {0xB62, 0xB63},-{0xB82, 0xB82}, {0xBBE, 0xBC2}, {0xBC6, 0xBC8}, {0xBCA, 0xBCD}, {0xBD7, 0xBD7}, {0xC00, 0xC04}, {0xC3E, 0xC44}, {0xC46, 0xC48}, {0xC4A, 0xC4D}, {0xC55, 0xC56}, {0xC62, 0xC63}, {0xC81, 0xC83},-{0xCBC, 0xCBC}, {0xCBE, 0xCC4}, {0xCC6, 0xCC8}, {0xCCA, 0xCCD}, {0xCD5, 0xCD6}, {0xCE2, 0xCE3}, {0xD00, 0xD03}, {0xD3B, 0xD3C}, {0xD3E, 0xD44}, {0xD46, 0xD48}, {0xD4A, 0xD4D}, {0xD57, 0xD57},-{0xD62, 0xD63}, {0xD81, 0xD83}, {0xDCA, 0xDCA}, {0xDCF, 0xDD4}, {0xDD6, 0xDD6}, {0xDD8, 0xDDF}, {0xDF2, 0xDF3}, {0xE31, 0xE31}, {0xE34, 0xE3A}, {0xE47, 0xE4E}, {0xEB1, 0xEB1}, {0xEB4, 0xEBC},-{0xEC8, 0xECD}, {0xF18, 0xF19}, {0xF35, 0xF35}, {0xF37, 0xF37}, {0xF39, 0xF39}, {0xF3E, 0xF3F}, {0xF71, 0xF84}, {0xF86, 0xF87}, {0xF8D, 0xF97}, {0xF99, 0xFBC}, {0xFC6, 0xFC6}, {0x102B, 0x103E},-{0x1056, 0x1059}, {0x105E, 0x1060}, {0x1062, 0x1064}, {0x1067, 0x106D}, {0x1071, 0x1074}, {0x1082, 0x108D}, {0x108F, 0x108F}, {0x109A, 0x109D}, {0x135D, 0x135F}, {0x1712, 0x1714}, {0x1732, 0x1734},-{0x1752, 0x1753}, {0x1772, 0x1773}, {0x17B4, 0x17D3}, {0x17DD, 0x17DD}, {0x180B, 0x180D}, {0x1885, 0x1886}, {0x18A9, 0x18A9}, {0x1920, 0x192B}, {0x1930, 0x193B}, {0x1A17, 0x1A1B}, {0x1A55, 0x1A5E},-{0x1A60, 0x1A7C}, {0x1A7F, 0x1A7F}, {0x1AB0, 0x1AC0}, {0x1B00, 0x1B04}, {0x1B34, 0x1B44}, {0x1B6B, 0x1B73}, {0x1B80, 0x1B82}, {0x1BA1, 0x1BAD}, {0x1BE6, 0x1BF3}, {0x1C24, 0x1C37}, {0x1CD0, 0x1CD2},-{0x1CD4, 0x1CE8}, {0x1CED, 0x1CED}, {0x1CF4, 0x1CF4}, {0x1CF7, 0x1CF9}, {0x1DC0, 0x1DF9}, {0x1DFB, 0x1DFF}, {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F}, {0x2DE0, 0x2DFF}, {0x302A, 0x302F},-{0x3099, 0x309A}, {0xA66F, 0xA672}, {0xA674, 0xA67D}, {0xA69E, 0xA69F}, {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA806, 0xA806}, {0xA80B, 0xA80B}, {0xA823, 0xA827}, {0xA82C, 0xA82C}, {0xA880, 0xA881},-{0xA8B4, 0xA8C5}, {0xA8E0, 0xA8F1}, {0xA8FF, 0xA8FF}, {0xA926, 0xA92D}, {0xA947, 0xA953}, {0xA980, 0xA983}, {0xA9B3, 0xA9C0}, {0xA9E5, 0xA9E5}, {0xAA29, 0xAA36}, {0xAA43, 0xAA43}, {0xAA4C, 0xAA4D},-{0xAA7B, 0xAA7D}, {0xAAB0, 0xAAB0}, {0xAAB2, 0xAAB4}, {0xAAB7, 0xAAB8}, {0xAABE, 0xAABF}, {0xAAC1, 0xAAC1}, {0xAAEB, 0xAAEF}, {0xAAF5, 0xAAF6}, {0xABE3, 0xABEA}, {0xABEC, 0xABED}, {0xFB1E, 0xFB1E},-{0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, {0x102E0, 0x102E0}, {0x10376, 0x1037A}, {0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F},-{0x10AE5, 0x10AE6}, {0x10D24, 0x10D27}, {0x10EAB, 0x10EAC}, {0x10F46, 0x10F50}, {0x11000, 0x11002}, {0x11038, 0x11046}, {0x1107F, 0x11082}, {0x110B0, 0x110BA}, {0x11100, 0x11102}, {0x11127, 0x11134},-{0x11145, 0x11146}, {0x11173, 0x11173}, {0x11180, 0x11182}, {0x111B3, 0x111C0}, {0x111C9, 0x111CC}, {0x111CE, 0x111CF}, {0x1122C, 0x11237}, {0x1123E, 0x1123E}, {0x112DF, 0x112EA}, {0x11300, 0x11303},-{0x1133B, 0x1133C}, {0x1133E, 0x11344}, {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11357, 0x11357}, {0x11362, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11435, 0x11446}, {0x1145E, 0x1145E},-{0x114B0, 0x114C3}, {0x115AF, 0x115B5}, {0x115B8, 0x115C0}, {0x115DC, 0x115DD}, {0x11630, 0x11640}, {0x116AB, 0x116B7}, {0x1171D, 0x1172B}, {0x1182C, 0x1183A}, {0x11930, 0x11935}, {0x11937, 0x11938},-{0x1193B, 0x1193E}, {0x11940, 0x11940}, {0x11942, 0x11943}, {0x119D1, 0x119D7}, {0x119DA, 0x119E0}, {0x119E4, 0x119E4}, {0x11A01, 0x11A0A}, {0x11A33, 0x11A39}, {0x11A3B, 0x11A3E}, {0x11A47, 0x11A47},-{0x11A51, 0x11A5B}, {0x11A8A, 0x11A99}, {0x11C2F, 0x11C36}, {0x11C38, 0x11C3F}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x11D31, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, {0x11D3F, 0x11D45},-{0x11D47, 0x11D47}, {0x11D8A, 0x11D8E}, {0x11D90, 0x11D91}, {0x11D93, 0x11D97}, {0x11EF3, 0x11EF6}, {0x16AF0, 0x16AF4}, {0x16B30, 0x16B36}, {0x16F4F, 0x16F4F}, {0x16F51, 0x16F87}, {0x16F8F, 0x16F92},-{0x16FE4, 0x16FE4}, {0x16FF0, 0x16FF1}, {0x1BC9D, 0x1BC9E}, {0x1D165, 0x1D169}, {0x1D16D, 0x1D172}, {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, {0x1DA00, 0x1DA36},-{0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75}, {0x1DA84, 0x1DA84}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E026, 0x1E02A},-{0x1E130, 0x1E136}, {0x1E2EC, 0x1E2EF}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A}, {0xE0100, 0xE01EF},-};--static const std::vector<std::pair<uint32_t, uint32_t>> punctuation_ranges = {-{0x21, 0x23}, {0x25, 0x2A}, {0x2C, 0x2F}, {0x3A, 0x3B}, {0x3F, 0x40}, {0x5B, 0x5D}, {0x5F, 0x5F}, {0x7B, 0x7B}, {0x7D, 0x7D}, {0xA1, 0xA1}, {0xA7, 0xA7}, {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB},-{0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387}, {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3}, {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D},-{0x61B, 0x61B}, {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9}, {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD}, {0xA76, 0xA76},-{0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4}, {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D}, {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA},-{0x104A, 0x104F}, {0x10FB, 0x10FB}, {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED}, {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A},-{0x1944, 0x1945}, {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF}, {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027},-{0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E}, {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF}, {0x2983, 0x2998},-{0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF}, {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003}, {0x3008, 0x3011}, {0x3014, 0x301F},-{0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0}, {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E}, {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF},-{0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC}, {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F}, {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F},-{0xFE10, 0xFE19}, {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B}, {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20},-{0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65}, {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F}, {0x10857, 0x10857},-{0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58}, {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C}, {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D},-{0x110BB, 0x110BC}, {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8}, {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D}, {0x112A9, 0x112A9},-{0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D}, {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C}, {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946},-{0x119E2, 0x119E2}, {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45}, {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474}, {0x16A6E, 0x16A6F},-{0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44}, {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B}, {0x1E95E, 0x1E95F},-};--static const std::vector<std::pair<uint32_t, uint32_t>> symbol_ranges = {-{0x24, 0x24}, {0x2B, 0x2B}, {0x3C, 0x3E}, {0x5E, 0x5E}, {0x60, 0x60}, {0x7C, 0x7C}, {0x7E, 0x7E}, {0xA2, 0xA6}, {0xA8, 0xA9}, {0xAC, 0xAC}, {0xAE, 0xB1}, {0xB4, 0xB4}, {0xB8, 0xB8}, {0xD7, 0xD7},-{0xF7, 0xF7}, {0x2C2, 0x2C5}, {0x2D2, 0x2DF}, {0x2E5, 0x2EB}, {0x2ED, 0x2ED}, {0x2EF, 0x2FF}, {0x375, 0x375}, {0x384, 0x385}, {0x3F6, 0x3F6}, {0x482, 0x482}, {0x58D, 0x58F}, {0x606, 0x608},-{0x60B, 0x60B}, {0x60E, 0x60F}, {0x6DE, 0x6DE}, {0x6E9, 0x6E9}, {0x6FD, 0x6FE}, {0x7F6, 0x7F6}, {0x7FE, 0x7FF}, {0x9F2, 0x9F3}, {0x9FA, 0x9FB}, {0xAF1, 0xAF1}, {0xB70, 0xB70}, {0xBF3, 0xBFA},-{0xC7F, 0xC7F}, {0xD4F, 0xD4F}, {0xD79, 0xD79}, {0xE3F, 0xE3F}, {0xF01, 0xF03}, {0xF13, 0xF13}, {0xF15, 0xF17}, {0xF1A, 0xF1F}, {0xF34, 0xF34}, {0xF36, 0xF36}, {0xF38, 0xF38}, {0xFBE, 0xFC5},-{0xFC7, 0xFCC}, {0xFCE, 0xFCF}, {0xFD5, 0xFD8}, {0x109E, 0x109F}, {0x1390, 0x1399}, {0x166D, 0x166D}, {0x17DB, 0x17DB}, {0x1940, 0x1940}, {0x19DE, 0x19FF}, {0x1B61, 0x1B6A}, {0x1B74, 0x1B7C},-{0x1FBD, 0x1FBD}, {0x1FBF, 0x1FC1}, {0x1FCD, 0x1FCF}, {0x1FDD, 0x1FDF}, {0x1FED, 0x1FEF}, {0x1FFD, 0x1FFE}, {0x2044, 0x2044}, {0x2052, 0x2052}, {0x207A, 0x207C}, {0x208A, 0x208C}, {0x20A0, 0x20BF},-{0x2100, 0x2101}, {0x2103, 0x2106}, {0x2108, 0x2109}, {0x2114, 0x2114}, {0x2116, 0x2118}, {0x211E, 0x2123}, {0x2125, 0x2125}, {0x2127, 0x2127}, {0x2129, 0x2129}, {0x212E, 0x212E}, {0x213A, 0x213B},-{0x2140, 0x2144}, {0x214A, 0x214D}, {0x214F, 0x214F}, {0x218A, 0x218B}, {0x2190, 0x2307}, {0x230C, 0x2328}, {0x232B, 0x2426}, {0x2440, 0x244A}, {0x249C, 0x24E9}, {0x2500, 0x2767}, {0x2794, 0x27C4},-{0x27C7, 0x27E5}, {0x27F0, 0x2982}, {0x2999, 0x29D7}, {0x29DC, 0x29FB}, {0x29FE, 0x2B73}, {0x2B76, 0x2B95}, {0x2B97, 0x2BFF}, {0x2CE5, 0x2CEA}, {0x2E50, 0x2E51}, {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3},-{0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, {0x3004, 0x3004}, {0x3012, 0x3013}, {0x3020, 0x3020}, {0x3036, 0x3037}, {0x303E, 0x303F}, {0x309B, 0x309C}, {0x3190, 0x3191}, {0x3196, 0x319F}, {0x31C0, 0x31E3},-{0x3200, 0x321E}, {0x322A, 0x3247}, {0x3250, 0x3250}, {0x3260, 0x327F}, {0x328A, 0x32B0}, {0x32C0, 0x33FF}, {0x4DC0, 0x4DFF}, {0xA490, 0xA4C6}, {0xA700, 0xA716}, {0xA720, 0xA721}, {0xA789, 0xA78A},-{0xA828, 0xA82B}, {0xA836, 0xA839}, {0xAA77, 0xAA79}, {0xAB5B, 0xAB5B}, {0xAB6A, 0xAB6B}, {0xFB29, 0xFB29}, {0xFBB2, 0xFBC1}, {0xFDFC, 0xFDFD}, {0xFE62, 0xFE62}, {0xFE64, 0xFE66}, {0xFE69, 0xFE69},-{0xFF04, 0xFF04}, {0xFF0B, 0xFF0B}, {0xFF1C, 0xFF1E}, {0xFF3E, 0xFF3E}, {0xFF40, 0xFF40}, {0xFF5C, 0xFF5C}, {0xFF5E, 0xFF5E}, {0xFFE0, 0xFFE6}, {0xFFE8, 0xFFEE}, {0xFFFC, 0xFFFD}, {0x10137, 0x1013F},-{0x10179, 0x10189}, {0x1018C, 0x1018E}, {0x10190, 0x1019C}, {0x101A0, 0x101A0}, {0x101D0, 0x101FC}, {0x10877, 0x10878}, {0x10AC8, 0x10AC8}, {0x1173F, 0x1173F}, {0x11FD5, 0x11FF1}, {0x16B3C, 0x16B3F},-{0x16B45, 0x16B45}, {0x1BC9C, 0x1BC9C}, {0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D164}, {0x1D16A, 0x1D16C}, {0x1D183, 0x1D184}, {0x1D18C, 0x1D1A9}, {0x1D1AE, 0x1D1E8}, {0x1D200, 0x1D241},-{0x1D245, 0x1D245}, {0x1D300, 0x1D356}, {0x1D6C1, 0x1D6C1}, {0x1D6DB, 0x1D6DB}, {0x1D6FB, 0x1D6FB}, {0x1D715, 0x1D715}, {0x1D735, 0x1D735}, {0x1D74F, 0x1D74F}, {0x1D76F, 0x1D76F}, {0x1D789, 0x1D789},-{0x1D7A9, 0x1D7A9}, {0x1D7C3, 0x1D7C3}, {0x1D800, 0x1D9FF}, {0x1DA37, 0x1DA3A}, {0x1DA6D, 0x1DA74}, {0x1DA76, 0x1DA83}, {0x1DA85, 0x1DA86}, {0x1E14F, 0x1E14F}, {0x1E2FF, 0x1E2FF}, {0x1ECAC, 0x1ECAC},-{0x1ECB0, 0x1ECB0}, {0x1ED2E, 0x1ED2E}, {0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F02B}, {0x1F030, 0x1F093}, {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, {0x1F0C1, 0x1F0CF}, {0x1F0D1, 0x1F0F5}, {0x1F10D, 0x1F1AD},-{0x1F1E6, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, {0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F6D7}, {0x1F6E0, 0x1F6EC}, {0x1F6F0, 0x1F6FC}, {0x1F700, 0x1F773}, {0x1F780, 0x1F7D8},-{0x1F7E0, 0x1F7EB}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0x1F8B0, 0x1F8B1}, {0x1F900, 0x1F978}, {0x1F97A, 0x1F9CB}, {0x1F9CD, 0x1FA53},-{0x1FA60, 0x1FA6D}, {0x1FA70, 0x1FA74}, {0x1FA78, 0x1FA7A}, {0x1FA80, 0x1FA86}, {0x1FA90, 0x1FAA8}, {0x1FAB0, 0x1FAB6}, {0x1FAC0, 0x1FAC2}, {0x1FAD0, 0x1FAD6}, {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA},-};--static const std::vector<std::pair<uint32_t, uint32_t>> control_ranges = {-{0x0, 0x8}, {0xE, 0x1B}, {0x7F, 0x84}, {0x86, 0x9F}, {0xAD, 0xAD}, {0x378, 0x379}, {0x380, 0x383}, {0x38B, 0x38B}, {0x38D, 0x38D}, {0x3A2, 0x3A2}, {0x530, 0x530}, {0x557, 0x558}, {0x58B, 0x58C},-{0x590, 0x590}, {0x5C8, 0x5CF}, {0x5EB, 0x5EE}, {0x5F5, 0x605}, {0x61C, 0x61D}, {0x6DD, 0x6DD}, {0x70E, 0x70F}, {0x74B, 0x74C}, {0x7B2, 0x7BF}, {0x7FB, 0x7FC}, {0x82E, 0x82F}, {0x83F, 0x83F},-{0x85C, 0x85D}, {0x85F, 0x85F}, {0x86B, 0x89F}, {0x8B5, 0x8B5}, {0x8C8, 0x8D2}, {0x8E2, 0x8E2}, {0x984, 0x984}, {0x98D, 0x98E}, {0x991, 0x992}, {0x9A9, 0x9A9}, {0x9B1, 0x9B1}, {0x9B3, 0x9B5},-{0x9BA, 0x9BB}, {0x9C5, 0x9C6}, {0x9C9, 0x9CA}, {0x9CF, 0x9D6}, {0x9D8, 0x9DB}, {0x9DE, 0x9DE}, {0x9E4, 0x9E5}, {0x9FF, 0xA00}, {0xA04, 0xA04}, {0xA0B, 0xA0E}, {0xA11, 0xA12}, {0xA29, 0xA29},-{0xA31, 0xA31}, {0xA34, 0xA34}, {0xA37, 0xA37}, {0xA3A, 0xA3B}, {0xA3D, 0xA3D}, {0xA43, 0xA46}, {0xA49, 0xA4A}, {0xA4E, 0xA50}, {0xA52, 0xA58}, {0xA5D, 0xA5D}, {0xA5F, 0xA65}, {0xA77, 0xA80},-{0xA84, 0xA84}, {0xA8E, 0xA8E}, {0xA92, 0xA92}, {0xAA9, 0xAA9}, {0xAB1, 0xAB1}, {0xAB4, 0xAB4}, {0xABA, 0xABB}, {0xAC6, 0xAC6}, {0xACA, 0xACA}, {0xACE, 0xACF}, {0xAD1, 0xADF}, {0xAE4, 0xAE5},-{0xAF2, 0xAF8}, {0xB00, 0xB00}, {0xB04, 0xB04}, {0xB0D, 0xB0E}, {0xB11, 0xB12}, {0xB29, 0xB29}, {0xB31, 0xB31}, {0xB34, 0xB34}, {0xB3A, 0xB3B}, {0xB45, 0xB46}, {0xB49, 0xB4A}, {0xB4E, 0xB54},-{0xB58, 0xB5B}, {0xB5E, 0xB5E}, {0xB64, 0xB65}, {0xB78, 0xB81}, {0xB84, 0xB84}, {0xB8B, 0xB8D}, {0xB91, 0xB91}, {0xB96, 0xB98}, {0xB9B, 0xB9B}, {0xB9D, 0xB9D}, {0xBA0, 0xBA2}, {0xBA5, 0xBA7},-{0xBAB, 0xBAD}, {0xBBA, 0xBBD}, {0xBC3, 0xBC5}, {0xBC9, 0xBC9}, {0xBCE, 0xBCF}, {0xBD1, 0xBD6}, {0xBD8, 0xBE5}, {0xBFB, 0xBFF}, {0xC0D, 0xC0D}, {0xC11, 0xC11}, {0xC29, 0xC29}, {0xC3A, 0xC3C},-{0xC45, 0xC45}, {0xC49, 0xC49}, {0xC4E, 0xC54}, {0xC57, 0xC57}, {0xC5B, 0xC5F}, {0xC64, 0xC65}, {0xC70, 0xC76}, {0xC8D, 0xC8D}, {0xC91, 0xC91}, {0xCA9, 0xCA9}, {0xCB4, 0xCB4}, {0xCBA, 0xCBB},-{0xCC5, 0xCC5}, {0xCC9, 0xCC9}, {0xCCE, 0xCD4}, {0xCD7, 0xCDD}, {0xCDF, 0xCDF}, {0xCE4, 0xCE5}, {0xCF0, 0xCF0}, {0xCF3, 0xCFF}, {0xD0D, 0xD0D}, {0xD11, 0xD11}, {0xD45, 0xD45}, {0xD49, 0xD49},-{0xD50, 0xD53}, {0xD64, 0xD65}, {0xD80, 0xD80}, {0xD84, 0xD84}, {0xD97, 0xD99}, {0xDB2, 0xDB2}, {0xDBC, 0xDBC}, {0xDBE, 0xDBF}, {0xDC7, 0xDC9}, {0xDCB, 0xDCE}, {0xDD5, 0xDD5}, {0xDD7, 0xDD7},-{0xDE0, 0xDE5}, {0xDF0, 0xDF1}, {0xDF5, 0xE00}, {0xE3B, 0xE3E}, {0xE5C, 0xE80}, {0xE83, 0xE83}, {0xE85, 0xE85}, {0xE8B, 0xE8B}, {0xEA4, 0xEA4}, {0xEA6, 0xEA6}, {0xEBE, 0xEBF}, {0xEC5, 0xEC5},-{0xEC7, 0xEC7}, {0xECE, 0xECF}, {0xEDA, 0xEDB}, {0xEE0, 0xEFF}, {0xF48, 0xF48}, {0xF6D, 0xF70}, {0xF98, 0xF98}, {0xFBD, 0xFBD}, {0xFCD, 0xFCD}, {0xFDB, 0xFFF}, {0x10C6, 0x10C6}, {0x10C8, 0x10CC},-{0x10CE, 0x10CF}, {0x1249, 0x1249}, {0x124E, 0x124F}, {0x1257, 0x1257}, {0x1259, 0x1259}, {0x125E, 0x125F}, {0x1289, 0x1289}, {0x128E, 0x128F}, {0x12B1, 0x12B1}, {0x12B6, 0x12B7}, {0x12BF, 0x12BF},-{0x12C1, 0x12C1}, {0x12C6, 0x12C7}, {0x12D7, 0x12D7}, {0x1311, 0x1311}, {0x1316, 0x1317}, {0x135B, 0x135C}, {0x137D, 0x137F}, {0x139A, 0x139F}, {0x13F6, 0x13F7}, {0x13FE, 0x13FF}, {0x169D, 0x169F},-{0x16F9, 0x16FF}, {0x170D, 0x170D}, {0x1715, 0x171F}, {0x1737, 0x173F}, {0x1754, 0x175F}, {0x176D, 0x176D}, {0x1771, 0x1771}, {0x1774, 0x177F}, {0x17DE, 0x17DF}, {0x17EA, 0x17EF}, {0x17FA, 0x17FF},-{0x180E, 0x180F}, {0x181A, 0x181F}, {0x1879, 0x187F}, {0x18AB, 0x18AF}, {0x18F6, 0x18FF}, {0x191F, 0x191F}, {0x192C, 0x192F}, {0x193C, 0x193F}, {0x1941, 0x1943}, {0x196E, 0x196F}, {0x1975, 0x197F},-{0x19AC, 0x19AF}, {0x19CA, 0x19CF}, {0x19DB, 0x19DD}, {0x1A1C, 0x1A1D}, {0x1A5F, 0x1A5F}, {0x1A7D, 0x1A7E}, {0x1A8A, 0x1A8F}, {0x1A9A, 0x1A9F}, {0x1AAE, 0x1AAF}, {0x1AC1, 0x1AFF}, {0x1B4C, 0x1B4F},-{0x1B7D, 0x1B7F}, {0x1BF4, 0x1BFB}, {0x1C38, 0x1C3A}, {0x1C4A, 0x1C4C}, {0x1C89, 0x1C8F}, {0x1CBB, 0x1CBC}, {0x1CC8, 0x1CCF}, {0x1CFB, 0x1CFF}, {0x1DFA, 0x1DFA}, {0x1F16, 0x1F17}, {0x1F1E, 0x1F1F},-{0x1F46, 0x1F47}, {0x1F4E, 0x1F4F}, {0x1F58, 0x1F58}, {0x1F5A, 0x1F5A}, {0x1F5C, 0x1F5C}, {0x1F5E, 0x1F5E}, {0x1F7E, 0x1F7F}, {0x1FB5, 0x1FB5}, {0x1FC5, 0x1FC5}, {0x1FD4, 0x1FD5}, {0x1FDC, 0x1FDC},-{0x1FF0, 0x1FF1}, {0x1FF5, 0x1FF5}, {0x1FFF, 0x1FFF}, {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x206F}, {0x2072, 0x2073}, {0x208F, 0x208F}, {0x209D, 0x209F}, {0x20C0, 0x20CF}, {0x20F1, 0x20FF},-{0x218C, 0x218F}, {0x2427, 0x243F}, {0x244B, 0x245F}, {0x2B74, 0x2B75}, {0x2B96, 0x2B96}, {0x2C2F, 0x2C2F}, {0x2C5F, 0x2C5F}, {0x2CF4, 0x2CF8}, {0x2D26, 0x2D26}, {0x2D28, 0x2D2C}, {0x2D2E, 0x2D2F},-{0x2D68, 0x2D6E}, {0x2D71, 0x2D7E}, {0x2D97, 0x2D9F}, {0x2DA7, 0x2DA7}, {0x2DAF, 0x2DAF}, {0x2DB7, 0x2DB7}, {0x2DBF, 0x2DBF}, {0x2DC7, 0x2DC7}, {0x2DCF, 0x2DCF}, {0x2DD7, 0x2DD7}, {0x2DDF, 0x2DDF},-{0x2E53, 0x2E7F}, {0x2E9A, 0x2E9A}, {0x2EF4, 0x2EFF}, {0x2FD6, 0x2FEF}, {0x2FFC, 0x2FFF}, {0x3040, 0x3040}, {0x3097, 0x3098}, {0x3100, 0x3104}, {0x3130, 0x3130}, {0x318F, 0x318F}, {0x31E4, 0x31EF},-{0x321F, 0x321F}, {0x9FFD, 0x9FFF}, {0xA48D, 0xA48F}, {0xA4C7, 0xA4CF}, {0xA62C, 0xA63F}, {0xA6F8, 0xA6FF}, {0xA7C0, 0xA7C1}, {0xA7CB, 0xA7F4}, {0xA82D, 0xA82F}, {0xA83A, 0xA83F}, {0xA878, 0xA87F},-{0xA8C6, 0xA8CD}, {0xA8DA, 0xA8DF}, {0xA954, 0xA95E}, {0xA97D, 0xA97F}, {0xA9CE, 0xA9CE}, {0xA9DA, 0xA9DD}, {0xA9FF, 0xA9FF}, {0xAA37, 0xAA3F}, {0xAA4E, 0xAA4F}, {0xAA5A, 0xAA5B}, {0xAAC3, 0xAADA},-{0xAAF7, 0xAB00}, {0xAB07, 0xAB08}, {0xAB0F, 0xAB10}, {0xAB17, 0xAB1F}, {0xAB27, 0xAB27}, {0xAB2F, 0xAB2F}, {0xAB6C, 0xAB6F}, {0xABEE, 0xABEF}, {0xABFA, 0xABFF}, {0xD7A4, 0xD7AF}, {0xD7C7, 0xD7CA},-{0xD7FC, 0xF8FF}, {0xFA6E, 0xFA6F}, {0xFADA, 0xFAFF}, {0xFB07, 0xFB12}, {0xFB18, 0xFB1C}, {0xFB37, 0xFB37}, {0xFB3D, 0xFB3D}, {0xFB3F, 0xFB3F}, {0xFB42, 0xFB42}, {0xFB45, 0xFB45}, {0xFBC2, 0xFBD2},-{0xFD40, 0xFD4F}, {0xFD90, 0xFD91}, {0xFDC8, 0xFDEF}, {0xFDFE, 0xFDFF}, {0xFE1A, 0xFE1F}, {0xFE53, 0xFE53}, {0xFE67, 0xFE67}, {0xFE6C, 0xFE6F}, {0xFE75, 0xFE75}, {0xFEFD, 0xFF00}, {0xFFBF, 0xFFC1},-{0xFFC8, 0xFFC9}, {0xFFD0, 0xFFD1}, {0xFFD8, 0xFFD9}, {0xFFDD, 0xFFDF}, {0xFFE7, 0xFFE7}, {0xFFEF, 0xFFFB}, {0xFFFE, 0xFFFF}, {0x1000C, 0x1000C}, {0x10027, 0x10027}, {0x1003B, 0x1003B},-{0x1003E, 0x1003E}, {0x1004E, 0x1004F}, {0x1005E, 0x1007F}, {0x100FB, 0x100FF}, {0x10103, 0x10106}, {0x10134, 0x10136}, {0x1018F, 0x1018F}, {0x1019D, 0x1019F}, {0x101A1, 0x101CF}, {0x101FE, 0x1027F},-{0x1029D, 0x1029F}, {0x102D1, 0x102DF}, {0x102FC, 0x102FF}, {0x10324, 0x1032C}, {0x1034B, 0x1034F}, {0x1037B, 0x1037F}, {0x1039E, 0x1039E}, {0x103C4, 0x103C7}, {0x103D6, 0x103FF}, {0x1049E, 0x1049F},-{0x104AA, 0x104AF}, {0x104D4, 0x104D7}, {0x104FC, 0x104FF}, {0x10528, 0x1052F}, {0x10564, 0x1056E}, {0x10570, 0x105FF}, {0x10737, 0x1073F}, {0x10756, 0x1075F}, {0x10768, 0x107FF}, {0x10806, 0x10807},-{0x10809, 0x10809}, {0x10836, 0x10836}, {0x10839, 0x1083B}, {0x1083D, 0x1083E}, {0x10856, 0x10856}, {0x1089F, 0x108A6}, {0x108B0, 0x108DF}, {0x108F3, 0x108F3}, {0x108F6, 0x108FA}, {0x1091C, 0x1091E},-{0x1093A, 0x1093E}, {0x10940, 0x1097F}, {0x109B8, 0x109BB}, {0x109D0, 0x109D1}, {0x10A04, 0x10A04}, {0x10A07, 0x10A0B}, {0x10A14, 0x10A14}, {0x10A18, 0x10A18}, {0x10A36, 0x10A37}, {0x10A3B, 0x10A3E},-{0x10A49, 0x10A4F}, {0x10A59, 0x10A5F}, {0x10AA0, 0x10ABF}, {0x10AE7, 0x10AEA}, {0x10AF7, 0x10AFF}, {0x10B36, 0x10B38}, {0x10B56, 0x10B57}, {0x10B73, 0x10B77}, {0x10B92, 0x10B98}, {0x10B9D, 0x10BA8},-{0x10BB0, 0x10BFF}, {0x10C49, 0x10C7F}, {0x10CB3, 0x10CBF}, {0x10CF3, 0x10CF9}, {0x10D28, 0x10D2F}, {0x10D3A, 0x10E5F}, {0x10E7F, 0x10E7F}, {0x10EAA, 0x10EAA}, {0x10EAE, 0x10EAF}, {0x10EB2, 0x10EFF},-{0x10F28, 0x10F2F}, {0x10F5A, 0x10FAF}, {0x10FCC, 0x10FDF}, {0x10FF7, 0x10FFF}, {0x1104E, 0x11051}, {0x11070, 0x1107E}, {0x110BD, 0x110BD}, {0x110C2, 0x110CF}, {0x110E9, 0x110EF}, {0x110FA, 0x110FF},-{0x11135, 0x11135}, {0x11148, 0x1114F}, {0x11177, 0x1117F}, {0x111E0, 0x111E0}, {0x111F5, 0x111FF}, {0x11212, 0x11212}, {0x1123F, 0x1127F}, {0x11287, 0x11287}, {0x11289, 0x11289}, {0x1128E, 0x1128E},-{0x1129E, 0x1129E}, {0x112AA, 0x112AF}, {0x112EB, 0x112EF}, {0x112FA, 0x112FF}, {0x11304, 0x11304}, {0x1130D, 0x1130E}, {0x11311, 0x11312}, {0x11329, 0x11329}, {0x11331, 0x11331}, {0x11334, 0x11334},-{0x1133A, 0x1133A}, {0x11345, 0x11346}, {0x11349, 0x1134A}, {0x1134E, 0x1134F}, {0x11351, 0x11356}, {0x11358, 0x1135C}, {0x11364, 0x11365}, {0x1136D, 0x1136F}, {0x11375, 0x113FF}, {0x1145C, 0x1145C},-{0x11462, 0x1147F}, {0x114C8, 0x114CF}, {0x114DA, 0x1157F}, {0x115B6, 0x115B7}, {0x115DE, 0x115FF}, {0x11645, 0x1164F}, {0x1165A, 0x1165F}, {0x1166D, 0x1167F}, {0x116B9, 0x116BF}, {0x116CA, 0x116FF},-{0x1171B, 0x1171C}, {0x1172C, 0x1172F}, {0x11740, 0x117FF}, {0x1183C, 0x1189F}, {0x118F3, 0x118FE}, {0x11907, 0x11908}, {0x1190A, 0x1190B}, {0x11914, 0x11914}, {0x11917, 0x11917}, {0x11936, 0x11936},-{0x11939, 0x1193A}, {0x11947, 0x1194F}, {0x1195A, 0x1199F}, {0x119A8, 0x119A9}, {0x119D8, 0x119D9}, {0x119E5, 0x119FF}, {0x11A48, 0x11A4F}, {0x11AA3, 0x11ABF}, {0x11AF9, 0x11BFF}, {0x11C09, 0x11C09},-{0x11C37, 0x11C37}, {0x11C46, 0x11C4F}, {0x11C6D, 0x11C6F}, {0x11C90, 0x11C91}, {0x11CA8, 0x11CA8}, {0x11CB7, 0x11CFF}, {0x11D07, 0x11D07}, {0x11D0A, 0x11D0A}, {0x11D37, 0x11D39}, {0x11D3B, 0x11D3B},-{0x11D3E, 0x11D3E}, {0x11D48, 0x11D4F}, {0x11D5A, 0x11D5F}, {0x11D66, 0x11D66}, {0x11D69, 0x11D69}, {0x11D8F, 0x11D8F}, {0x11D92, 0x11D92}, {0x11D99, 0x11D9F}, {0x11DAA, 0x11EDF}, {0x11EF9, 0x11FAF},-{0x11FB1, 0x11FBF}, {0x11FF2, 0x11FFE}, {0x1239A, 0x123FF}, {0x1246F, 0x1246F}, {0x12475, 0x1247F}, {0x12544, 0x12FFF}, {0x1342F, 0x143FF}, {0x14647, 0x167FF}, {0x16A39, 0x16A3F}, {0x16A5F, 0x16A5F},-{0x16A6A, 0x16A6D}, {0x16A70, 0x16ACF}, {0x16AEE, 0x16AEF}, {0x16AF6, 0x16AFF}, {0x16B46, 0x16B4F}, {0x16B5A, 0x16B5A}, {0x16B62, 0x16B62}, {0x16B78, 0x16B7C}, {0x16B90, 0x16E3F}, {0x16E9B, 0x16EFF},-{0x16F4B, 0x16F4E}, {0x16F88, 0x16F8E}, {0x16FA0, 0x16FDF}, {0x16FE5, 0x16FEF}, {0x16FF2, 0x16FFF}, {0x187F8, 0x187FF}, {0x18CD6, 0x18CFF}, {0x18D09, 0x1AFFF}, {0x1B11F, 0x1B14F}, {0x1B153, 0x1B163},-{0x1B168, 0x1B16F}, {0x1B2FC, 0x1BBFF}, {0x1BC6B, 0x1BC6F}, {0x1BC7D, 0x1BC7F}, {0x1BC89, 0x1BC8F}, {0x1BC9A, 0x1BC9B}, {0x1BCA0, 0x1CFFF}, {0x1D0F6, 0x1D0FF}, {0x1D127, 0x1D128}, {0x1D173, 0x1D17A},-{0x1D1E9, 0x1D1FF}, {0x1D246, 0x1D2DF}, {0x1D2F4, 0x1D2FF}, {0x1D357, 0x1D35F}, {0x1D379, 0x1D3FF}, {0x1D455, 0x1D455}, {0x1D49D, 0x1D49D}, {0x1D4A0, 0x1D4A1}, {0x1D4A3, 0x1D4A4}, {0x1D4A7, 0x1D4A8},-{0x1D4AD, 0x1D4AD}, {0x1D4BA, 0x1D4BA}, {0x1D4BC, 0x1D4BC}, {0x1D4C4, 0x1D4C4}, {0x1D506, 0x1D506}, {0x1D50B, 0x1D50C}, {0x1D515, 0x1D515}, {0x1D51D, 0x1D51D}, {0x1D53A, 0x1D53A}, {0x1D53F, 0x1D53F},-{0x1D545, 0x1D545}, {0x1D547, 0x1D549}, {0x1D551, 0x1D551}, {0x1D6A6, 0x1D6A7}, {0x1D7CC, 0x1D7CD}, {0x1DA8C, 0x1DA9A}, {0x1DAA0, 0x1DAA0}, {0x1DAB0, 0x1DFFF}, {0x1E007, 0x1E007}, {0x1E019, 0x1E01A},-{0x1E022, 0x1E022}, {0x1E025, 0x1E025}, {0x1E02B, 0x1E0FF}, {0x1E12D, 0x1E12F}, {0x1E13E, 0x1E13F}, {0x1E14A, 0x1E14D}, {0x1E150, 0x1E2BF}, {0x1E2FA, 0x1E2FE}, {0x1E300, 0x1E7FF}, {0x1E8C5, 0x1E8C6},-{0x1E8D7, 0x1E8FF}, {0x1E94C, 0x1E94F}, {0x1E95A, 0x1E95D}, {0x1E960, 0x1EC70}, {0x1ECB5, 0x1ED00}, {0x1ED3E, 0x1EDFF}, {0x1EE04, 0x1EE04}, {0x1EE20, 0x1EE20}, {0x1EE23, 0x1EE23}, {0x1EE25, 0x1EE26},-{0x1EE28, 0x1EE28}, {0x1EE33, 0x1EE33}, {0x1EE38, 0x1EE38}, {0x1EE3A, 0x1EE3A}, {0x1EE3C, 0x1EE41}, {0x1EE43, 0x1EE46}, {0x1EE48, 0x1EE48}, {0x1EE4A, 0x1EE4A}, {0x1EE4C, 0x1EE4C}, {0x1EE50, 0x1EE50},-{0x1EE53, 0x1EE53}, {0x1EE55, 0x1EE56}, {0x1EE58, 0x1EE58}, {0x1EE5A, 0x1EE5A}, {0x1EE5C, 0x1EE5C}, {0x1EE5E, 0x1EE5E}, {0x1EE60, 0x1EE60}, {0x1EE63, 0x1EE63}, {0x1EE65, 0x1EE66}, {0x1EE6B, 0x1EE6B},-{0x1EE73, 0x1EE73}, {0x1EE78, 0x1EE78}, {0x1EE7D, 0x1EE7D}, {0x1EE7F, 0x1EE7F}, {0x1EE8A, 0x1EE8A}, {0x1EE9C, 0x1EEA0}, {0x1EEA4, 0x1EEA4}, {0x1EEAA, 0x1EEAA}, {0x1EEBC, 0x1EEEF}, {0x1EEF2, 0x1EFFF},-{0x1F02C, 0x1F02F}, {0x1F094, 0x1F09F}, {0x1F0AF, 0x1F0B0}, {0x1F0C0, 0x1F0C0}, {0x1F0D0, 0x1F0D0}, {0x1F0F6, 0x1F0FF}, {0x1F1AE, 0x1F1E5}, {0x1F203, 0x1F20F}, {0x1F23C, 0x1F23F}, {0x1F249, 0x1F24F},-{0x1F252, 0x1F25F}, {0x1F266, 0x1F2FF}, {0x1F6D8, 0x1F6DF}, {0x1F6ED, 0x1F6EF}, {0x1F6FD, 0x1F6FF}, {0x1F774, 0x1F77F}, {0x1F7D9, 0x1F7DF}, {0x1F7EC, 0x1F7FF}, {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F},-{0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F8AF}, {0x1F8B2, 0x1F8FF}, {0x1F979, 0x1F979}, {0x1F9CC, 0x1F9CC}, {0x1FA54, 0x1FA5F}, {0x1FA6E, 0x1FA6F}, {0x1FA75, 0x1FA77}, {0x1FA7B, 0x1FA7F},-{0x1FA87, 0x1FA8F}, {0x1FAA9, 0x1FAAF}, {0x1FAB7, 0x1FABF}, {0x1FAC3, 0x1FACF}, {0x1FAD7, 0x1FAFF}, {0x1FB93, 0x1FB93}, {0x1FBCB, 0x1FBEF}, {0x1FBFA, 0x1FFFF}, {0x2A6DE, 0x2A6FF}, {0x2B735, 0x2B73F},-{0x2B81E, 0x2B81F}, {0x2CEA2, 0x2CEAF}, {0x2EBE1, 0x2F7FF}, {0x2FA1E, 0x2FFFF}, {0x3134B, 0xE00FF}, {0xE01F0, 0x10FFFF},-};--static std::string codepoint_to_utf8(uint32_t cp) {- std::string result;- if (/* 0x00 <= cp && */ cp <= 0x7f) {- result.push_back(cp);- }- else if (0x80 <= cp && cp <= 0x7ff) {- result.push_back(0xc0 | ((cp >> 6) & 0x1f));- result.push_back(0x80 | (cp & 0x3f));- }- else if (0x800 <= cp && cp <= 0xffff) {- result.push_back(0xe0 | ((cp >> 12) & 0x0f));- result.push_back(0x80 | ((cp >> 6) & 0x3f));- result.push_back(0x80 | (cp & 0x3f));- }- else if (0x10000 <= cp && cp <= 0x10ffff) {- result.push_back(0xf0 | ((cp >> 18) & 0x07));- result.push_back(0x80 | ((cp >> 12) & 0x3f));- result.push_back(0x80 | ((cp >> 6) & 0x3f));- result.push_back(0x80 | (cp & 0x3f));- }- else {- throw std::invalid_argument("invalid codepoint");- }- return result;-}--static std::string codepoints_to_utf8(const std::vector<uint32_t> & cps) {- std::string result;- for (size_t i = 0; i < cps.size(); ++i) {- result.append(codepoint_to_utf8(cps[i]));- }- return result;-}--static uint32_t codepoint_from_utf8(const std::string & utf8, size_t & offset) {- assert(offset < utf8.size());- if (!(utf8[offset + 0] & 0x80)) {- auto result = utf8[offset + 0];- offset += 1;- return result;- }- else if (!(utf8[offset + 0] & 0x40)) {- throw std::invalid_argument("invalid character");- }- else if (!(utf8[offset + 0] & 0x20)) {- if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80))- throw std::invalid_argument("invalid character");- auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);- offset += 2;- return result;- }- else if (!(utf8[offset + 0] & 0x10)) {- if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80))- throw std::invalid_argument("invalid character");- auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);- offset += 3;- return result;- }- else if (!(utf8[offset + 0] & 0x08)) {- if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80))- throw std::invalid_argument("invalid character");- auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);- offset += 4;- return result;- }- throw std::invalid_argument("invalid string");-}--static std::vector<uint32_t> codepoints_from_utf8(const std::string & utf8) {- std::vector<uint32_t> result;- size_t offset = 0;- while (offset < utf8.size()) {- result.push_back(codepoint_from_utf8(utf8, offset));- }- return result;-}--static std::vector<uint16_t> codepoint_to_utf16(uint32_t cp) {- std::vector<uint16_t> result;- if (/* 0x0000 <= cp && */ cp <= 0xffff) {- result.emplace_back(cp);- }- else if (0x10000 <= cp && cp <= 0x10ffff) {- result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));- result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));- }- else {- throw std::invalid_argument("invalid codepoint");- }- return result;-}--static std::vector<uint16_t> codepoints_to_utf16(const std::vector<uint32_t> & cps) {- std::vector<uint16_t> result;- for (size_t i = 0; i < cps.size(); ++i) {- auto temp = codepoint_to_utf16(cps[i]);- result.insert(result.end(), temp.begin(), temp.end());- }- return result;-}--static uint32_t codepoint_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {- assert(offset < utf16.size());- if (((utf16[0] >> 10) << 10) != 0xd800) {- auto result = utf16[offset + 0];- offset += 1;- return result;- }- else {- if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00))- throw std::invalid_argument("invalid character");- auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));- offset += 2;- return result;- }- throw std::invalid_argument("invalid string");-}--static std::vector<uint32_t> codepoints_from_utf16(const std::vector<uint16_t> & utf16) {- std::vector<uint32_t> result;- size_t offset = 0;- while (offset < utf16.size())- result.push_back(codepoint_from_utf16(utf16, offset));- return result;-}#define CODEPOINT_TYPE_UNIDENTIFIED 0-#define CODEPOINT_TYPE_DIGIT 1-#define CODEPOINT_TYPE_LETTER 2-#define CODEPOINT_TYPE_WHITESPACE 3-#define CODEPOINT_TYPE_ACCENT_MARK 4-#define CODEPOINT_TYPE_PUNCTUATION 5-#define CODEPOINT_TYPE_SYMBOL 6-#define CODEPOINT_TYPE_CONTROL 7--static std::unordered_map<uint32_t, int> codepoint_type_map() {- std::unordered_map<uint32_t, int> codepoint_types;- for (auto p : digit_ranges) {- for(auto i = p.first; i <= p.second; ++ i)- codepoint_types[i] = CODEPOINT_TYPE_DIGIT;- }- for(auto p : letter_ranges) {- for(auto i = p.first; i <= p.second; ++ i)- codepoint_types[i] = CODEPOINT_TYPE_LETTER;- }- for(auto p : whitespace_ranges) {- for(auto i = p.first; i <= p.second; ++ i)- codepoint_types[i] = CODEPOINT_TYPE_WHITESPACE;- }- for(auto p : accent_mark_ranges) {- for(auto i = p.first; i <= p.second; ++ i)- codepoint_types[i] = CODEPOINT_TYPE_ACCENT_MARK;- }- for(auto p : punctuation_ranges) {- for(auto i = p.first; i <= p.second; ++ i)- codepoint_types[i] = CODEPOINT_TYPE_PUNCTUATION;- }- for (auto p : symbol_ranges) {- for (auto i = p.first; i <= p.second; ++i)- codepoint_types[i] = CODEPOINT_TYPE_SYMBOL;- }- for(auto p : control_ranges) {- for(auto i = p.first; i <= p.second; ++ i)- codepoint_types[i] = CODEPOINT_TYPE_CONTROL;- }- return codepoint_types;-}--static int codepoint_type(uint32_t cp) {- static std::unordered_map<uint32_t, int> codepoint_types = codepoint_type_map();- return codepoint_types[cp];-}--static int codepoint_type(const std::string & utf8) {- if (utf8.length() == 0)- return CODEPOINT_TYPE_UNIDENTIFIED;- size_t offset = 0;- return codepoint_type(codepoint_from_utf8(utf8, offset));-}+#define CODEPOINT_TYPE_DIGIT 1+#define CODEPOINT_TYPE_LETTER 2+#define CODEPOINT_TYPE_WHITESPACE 3+#define CODEPOINT_TYPE_ACCENT_MARK 4+#define CODEPOINT_TYPE_PUNCTUATION 5+#define CODEPOINT_TYPE_SYMBOL 6+#define CODEPOINT_TYPE_CONTROL 7-static std::unordered_map<uint8_t, std::string> bytes_to_unicode_map_bpe() {- std::unordered_map<uint8_t, std::string> map;- for (int ch = u'!'; ch <= u'~'; ++ch) {- assert(0 <= ch && ch < 256);- map[ch] = codepoint_to_utf8(ch);- }- for (int ch = u'¡'; ch <= u'¬'; ++ch) {- assert(0 <= ch && ch < 256);- map[ch] = codepoint_to_utf8(ch);- }- for (int ch = u'®'; ch <= u'ÿ'; ++ch) {- assert(0 <= ch && ch < 256);- map[ch] = codepoint_to_utf8(ch);- }- auto n = 0;- for (int ch = 0; ch < 256; ++ch) {- if (map.find(ch) == map.end()) {- map[ch] = codepoint_to_utf8(256 + n);- ++n;- }- }- return map;-}+std::string unicode_cpt_to_utf8(uint32_t cp);+std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);-static std::string bytes_to_unicode_bpe(uint8_t byte) {- static std::unordered_map<uint8_t, std::string> map = bytes_to_unicode_map_bpe();- return map.at(byte);-}+std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);-static std::unordered_map<std::string, uint8_t> unicode_to_bytes_map_bpe() {- std::unordered_map<std::string, uint8_t> map;- for (int ch = u'!'; ch <= u'~'; ++ch) {- assert(0 <= ch && ch < 256);- map[codepoint_to_utf8(ch)] = ch;- }- for (int ch = u'¡'; ch <= u'¬'; ++ch) {- assert(0 <= ch && ch < 256);- map[codepoint_to_utf8(ch)] = ch;- }- for (int ch = u'®'; ch <= u'ÿ'; ++ch) {- assert(0 <= ch && ch < 256);- map[codepoint_to_utf8(ch)] = ch;- }- auto n = 0;- for (int ch = 0; ch < 256; ++ch) {- if (map.find(codepoint_to_utf8(ch)) == map.end()) {- map[codepoint_to_utf8(256 + n)] = ch;- ++n;- }- }- return map;-}+int unicode_cpt_type(uint32_t cp);+int unicode_cpt_type(const std::string & utf8);-static uint8_t unicode_to_bytes_bpe(const std::string & utf8) {- static std::unordered_map<std::string, uint8_t> map = unicode_to_bytes_map_bpe();- return map.at(utf8);-}+std::string unicode_byte_to_utf8(uint8_t byte);+uint8_t unicode_utf8_to_byte(const std::string & utf8);diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex e49dbf57..2e59c795 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -18,6 +18,7 @@#include "llama.cpp/ggml-backend-impl.h"#include "llama.cpp/ggml-cuda.h"#include "llama.cpp/ggml-metal.h"+#include "llamafile/llamafile.h"#include "llamafile/log.h"#include "llamafile/x.h"#include <assert.h>@@ -45,6 +46,7 @@ __static_yoink("llamafile/llamafile.h");__static_yoink("llama.cpp/ggml-cuda.h");__static_yoink("llama.cpp/ggml-alloc.h");__static_yoink("llama.cpp/ggml-cuda.cu");+__static_yoink("llama.cpp/ggml-common.h");__static_yoink("llama.cpp/ggml-backend.h");__static_yoink("llama.cpp/ggml-backend-impl.h");@@ -56,7 +58,7 @@ __static_yoink("llama.cpp/ggml-backend-impl.h");#define NVCC_LIBS BLAS_ONLY("-lcublas"), "-lcuda"#define NVCC_FLAGS \- "--shared", "--forward-unknown-to-host-compiler", "-use_fast_math", "--compiler-options", \+ "--shared", "--forward-unknown-to-host-compiler", "--compiler-options", \(!IsWindows() ? (!IsAarch64() ? "-fPIC -O3 -march=native -mtune=native" \: "-fPIC -O3 -march=native -mtune=native -ffixed-x28") \: "/nologo /EHsc /O2 /GR /MT"), \@@ -77,6 +79,7 @@ static const struct Source {{"/zip/llama.cpp/ggml-impl.h", "ggml-impl.h"},{"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},{"/zip/llama.cpp/ggml-alloc.h", "ggml-alloc.h"},+ {"/zip/llama.cpp/ggml-common.h", "ggml-common.h"},{"/zip/llama.cpp/ggml-backend.h", "ggml-backend.h"},{"/zip/llama.cpp/ggml-backend-impl.h", "ggml-backend-impl.h"},{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last@@ -88,21 +91,14 @@ static struct Cuda {bool supported;atomic_uint once;typeof(ggml_cuda_link) *GGML_CALL link;- typeof(ggml_init_cublas) *GGML_CALL init;- typeof(ggml_cublas_loaded) *GGML_CALL loaded;- typeof(ggml_cuda_host_free) *GGML_CALL host_free;- typeof(ggml_cuda_host_malloc) *GGML_CALL host_malloc;- typeof(ggml_cuda_can_mul_mat) *GGML_CALL can_mul_mat;- typeof(ggml_cuda_compute_forward) *GGML_CALL compute_forward;- typeof(ggml_cuda_get_device_count) *GGML_CALL get_device_count;- typeof(ggml_cuda_get_device_description) *GGML_CALL get_device_description;typeof(ggml_backend_cuda_buffer_type) *GGML_CALL buffer_type;typeof(ggml_backend_cuda_host_buffer_type) *GGML_CALL host_buffer_type;typeof(ggml_backend_cuda_init) *GGML_CALL backend_init;- typeof(ggml_backend_cuda_get_device_count) *GGML_CALL get_device_count2;typeof(ggml_backend_cuda_split_buffer_type) *GGML_CALL split_buffer_type;typeof(ggml_backend_cuda_reg_devices) *GGML_CALL reg_devices;typeof(ggml_backend_cuda_get_device_memory) *GGML_CALL get_device_memory;+ typeof(ggml_backend_cuda_get_device_count) *GGML_CALL get_device_count;+ typeof(ggml_backend_cuda_unregister_host_buffer) *GGML_CALL unreg_host_buf;} ggml_cuda;static const char *Dlerror(void) {@@ -534,7 +530,6 @@ static bool compile_amd_unix(const char *dso, const char *src, const char *tmpdsoffload_arch,"-march=native","-mtune=native",- "-use_fast_math","-DGGML_BUILD=1","-DGGML_SHARED=1","-Wno-return-type",@@ -648,21 +643,14 @@ static bool link_cuda_dso(const char *dso, const char *dir) {// import functionsbool ok = true;ok &= !!(ggml_cuda.link = imp(lib, "ggml_cuda_link"));- ok &= !!(ggml_cuda.init = imp(lib, "ggml_init_cublas"));- ok &= !!(ggml_cuda.loaded = imp(lib, "ggml_cublas_loaded"));- ok &= !!(ggml_cuda.host_free = imp(lib, "ggml_cuda_host_free"));- ok &= !!(ggml_cuda.host_malloc = imp(lib, "ggml_cuda_host_malloc"));- ok &= !!(ggml_cuda.can_mul_mat = imp(lib, "ggml_cuda_can_mul_mat"));- ok &= !!(ggml_cuda.compute_forward = imp(lib, "ggml_cuda_compute_forward"));- ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_cuda_get_device_count"));- ok &= !!(ggml_cuda.get_device_description = imp(lib, "ggml_cuda_get_device_description"));ok &= !!(ggml_cuda.host_buffer_type = imp(lib, "ggml_backend_cuda_host_buffer_type"));ok &= !!(ggml_cuda.buffer_type = imp(lib, "ggml_backend_cuda_buffer_type"));ok &= !!(ggml_cuda.backend_init = imp(lib, "ggml_backend_cuda_init"));- ok &= !!(ggml_cuda.get_device_count2 = imp(lib, "ggml_backend_cuda_get_device_count"));ok &= !!(ggml_cuda.split_buffer_type = imp(lib, "ggml_backend_cuda_split_buffer_type"));ok &= !!(ggml_cuda.reg_devices = imp(lib, "ggml_backend_cuda_reg_devices"));ok &= !!(ggml_cuda.get_device_memory = imp(lib, "ggml_backend_cuda_get_device_memory"));+ ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_backend_cuda_get_device_count"));+ ok &= !!(ggml_cuda.unreg_host_buf = imp(lib, "ggml_backend_cuda_unregister_host_buffer"));if (!ok) {tinylog(__func__, ": error: not all cuda symbols could be imported\n", NULL);cosmo_dlclose(lib);@@ -670,14 +658,11 @@ static bool link_cuda_dso(const char *dso, const char *dir) {}// ask the library if actual gpu devices exist- ggml_cuda.link(ggml_backend_api());- tinylog(__func__, ": GPU support linked\n", NULL);- ggml_cuda.init();- if (ggml_cuda.loaded()) {+ if (ggml_cuda.link(ggml_backend_api())) {tinylog(__func__, ": GPU support loaded\n", NULL);return true;} else {- tinylog(__func__, ": GPU support not possible\n", NULL);+ tinylog(__func__, ": No GPU devices found\n", NULL);cosmo_dlclose(lib);return false;}@@ -884,57 +869,6 @@ bool llamafile_has_cuda(void) {return ggml_cuda.supported;}-GGML_CALL void ggml_init_cublas(void) {- if (!llamafile_has_cuda())- return;- return ggml_cuda.init();-}--GGML_CALL bool ggml_cublas_loaded(void) {- if (!llamafile_has_cuda())- return false;- return ggml_cuda.loaded();-}--GGML_CALL void *ggml_cuda_host_malloc(size_t n) {- if (!llamafile_has_cuda())- return NULL;- return ggml_cuda.host_malloc(n);-}--GGML_CALL void ggml_cuda_host_free(void *data) {- if (!llamafile_has_cuda())- return;- return ggml_cuda.host_free(data);-}--GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0, const struct ggml_tensor *src1,- struct ggml_tensor *dst) {- if (!llamafile_has_cuda())- return false;- return ggml_cuda.can_mul_mat(src0, src1, dst);-}--GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params *params,- struct ggml_tensor *tensor) {- if (!llamafile_has_cuda())- return false;- return ggml_cuda.compute_forward(params, tensor);-}--GGML_CALL int ggml_cuda_get_device_count(void) {- if (!llamafile_has_cuda())- return 0;- return ggml_cuda.get_device_count();-}--GGML_CALL void ggml_cuda_get_device_description(int device, char *description,- size_t description_size) {- if (!llamafile_has_cuda())- return;- return ggml_cuda.get_device_description(device, description, description_size);-}-GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {if (!llamafile_has_cuda())return 0;@@ -953,12 +887,6 @@ GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {return ggml_cuda.backend_init(device);}-GGML_CALL int ggml_backend_cuda_get_device_count(void) {- if (!llamafile_has_cuda())- return 0;- return ggml_cuda.get_device_count();-}-GGML_CALL ggml_backend_buffer_type_tggml_backend_cuda_split_buffer_type(const float *tensor_split) {if (!llamafile_has_cuda())@@ -977,3 +905,15 @@ GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t *free, sizreturn;return ggml_cuda.get_device_memory(device, free, total);}++GGML_CALL int ggml_backend_cuda_get_device_count(void) {+ if (!llamafile_has_cuda())+ return 0;+ return ggml_cuda.get_device_count();+}++GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void *buffer) {+ if (!llamafile_has_cuda())+ return;+ return ggml_cuda.unreg_host_buf(buffer);+}diff --git a/llamafile/metal.c b/llamafile/metal.cindex 82ed828c..2cdd3f92 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -38,6 +38,7 @@ __static_yoink("llama.cpp/ggml-impl.h");__static_yoink("llama.cpp/ggml-alloc.h");__static_yoink("llama.cpp/ggml-metal.m");__static_yoink("llama.cpp/ggml-metal.h");+__static_yoink("llama.cpp/ggml-common.h");__static_yoink("llama.cpp/ggml-quants.h");__static_yoink("llama.cpp/ggml-backend.h");__static_yoink("llama.cpp/ggml-metal.metal");@@ -52,6 +53,7 @@ static const struct Source {{"/zip/llama.cpp/ggml-impl.h", "ggml-impl.h"},{"/zip/llama.cpp/ggml-metal.h", "ggml-metal.h"},{"/zip/llama.cpp/ggml-alloc.h", "ggml-alloc.h"},+ {"/zip/llama.cpp/ggml-common.h", "ggml-common.h"},{"/zip/llama.cpp/ggml-quants.h", "ggml-quants.h"},{"/zip/llama.cpp/ggml-backend.h", "ggml-backend.h"},{"/zip/llama.cpp/ggml-metal.metal", "ggml-metal.metal"},diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 80b21007..36068f8c 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -15,6 +15,7 @@// See the License for the specific language governing permissions and// limitations under the License.+#include "llama.cpp/ggml-impl.h"#include "llama.cpp/ggml-quants.h"#include "llamafile.h"#include <assert.h>diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex d59f6db4..49e4edcd 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -76,13 +76,17 @@ inline bool isone(float x) {}template <typename T, typename TA, typename TB>-__device__ __forceinline__ void kahan(T *sum, T *err, TA a, TB b) {+__device__ __forceinline__ void madd(T *tally, T *kahan, TA a, TB b) {T x = a;T y = b;- T z = x * y - *err;- T t = *sum + z;- *err = (t - *sum) - z;- *sum = t;+#ifdef __FAST_MATH__+ *tally += x * y;+#else+ T z = x * y - *kahan;+ T t = *tally + z;+ *kahan = (t - *tally) - z;+ *tally = t;+#endif}////////////////////////////////////////////////////////////////////////////////////////////////////@@ -102,7 +106,7 @@ static __device__ void matvec(int m, int k, const SRC *A, int lda, const SRC *B,for (int l = threadIdx.x; l < k; l += WARPSIZE) {WORD b = B[l];for (int j = 0; j < WARPSIZE; ++j)- kahan(&Ct[j], &Ce[j], A[lda * (i + j) + l], b);+ madd(&Ct[j], &Ce[j], A[lda * (i + j) + l], b);}for (int j = 0; j < WARPSIZE; ++j) {WORD c = warpSum(Ct[j]);@@ -201,7 +205,7 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperatBt[h] = Bs[BN * l + tj + h];for (int j = 0; j < TM; ++j)for (int h = 0; h < TN; ++h)- kahan(&Ct[TN * j + h], &Ce[TN * j + h], At[j], Bt[h]);+ madd(&Ct[TN * j + h], &Ce[TN * j + h], At[j], Bt[h]);}__syncthreads();@@ -323,9 +327,9 @@ static __device__ void matmul_warp2d(tinyblasOperation_t aT, //for (int jj = 0; jj < WNI; ++jj)for (int i = 0; i < TM; ++i)for (int j = 0; j < TN; ++j)- kahan(&Ct[(WNI * TN) * (TM * ii + i) + (TN * jj) + j],- &Ce[(WNI * TN) * (TM * ii + i) + (TN * jj) + j], Ar[TM * ii + i],- Br[TN * jj + j]);+ madd(&Ct[(WNI * TN) * (TM * ii + i) + (TN * jj) + j],+ &Ce[(WNI * TN) * (TM * ii + i) + (TN * jj) + j], Ar[TM * ii + i],+ Br[TN * jj + j]);}__syncthreads();